diff --git a/CMakeLists.txt b/CMakeLists.txt index 592d16a88..ff59be5fa 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,7 @@ project(openGauss) if(POLICY CMP0068) cmake_policy(SET CMP0068 NEW) endif() - +//This is a test if(POLICY CMP0075) cmake_policy(SET CMP0075 NEW) endif() diff --git a/src/gausskernel/runtime/codegen/gscodegen.cpp b/src/gausskernel/runtime/codegen/gscodegen.cpp index 7e7b250fe..c95ad8277 100644 --- a/src/gausskernel/runtime/codegen/gscodegen.cpp +++ b/src/gausskernel/runtime/codegen/gscodegen.cpp @@ -351,7 +351,16 @@ void GsCodeGen::loadIRFile() if (NULL != exec_path && strcmp(exec_path, "\0") != 0) { char* exec_path_r = realpath(exec_path, NULL); if (exec_path_r) { - appendStringInfo(filename, "%s/share/llvmir/GaussDB_expr.ir", exec_path_r); + char *llvmIrFilePath = "share/llvmir/GaussDB_expr.ir"; +#if (!defined(ENABLE_MULTIPLE_NODES)) && (!defined(ENABLE_PRIVATEGAUSS)) + if (u_sess->attr.attr_sql.whale || u_sess->attr.attr_sql.dolphin) { + int id = GetCustomParserId(); + if (id >= 0 && g_instance.llvmIrFilePath[id] != NULL) { + llvmIrFilePath = g_instance.llvmIrFilePath[id]; + } + } +#endif + appendStringInfo(filename, "%s/%s", exec_path_r, llvmIrFilePath); check_backend_env(exec_path); free(exec_path_r); } else { @@ -380,7 +389,15 @@ void GsCodeGen::loadIRFile() pfree_ext(filename->data); pfree_ext(filename); } - +/* function name:compileCurrentModule + function purpose:Compile the current llvm module and generate machine code + input: enable_jitcache indicates whether to enable JIT caching + output:none + note:none + annotator:liushifa + annotate time:2023/09/23 16:24:35 + contact:3325287047@qq.com +*/ void GsCodeGen::compileCurrentModule(bool enable_jitcache) { /* m_currentModule == NULL when we hit a cache */ @@ -613,7 +630,15 @@ bool GsCodeGen::verifyFunction(Function* fn) } return true; } - +/* function name:FinalizeFunction + function purpose:Complete the final processing steps of the llvm function + input: + output:none + note:none + annotator:liushifa + annotate time:2023/09/23 16:30:43 + contact:3325287047@qq.com +*/ void GsCodeGen::FinalizeFunction(Function* function, int plan_node_id) { bool is_valid = false; @@ -1027,7 +1052,8 @@ void CodeGenThreadInitialize() bool CodeGenThreadObjectReady() { - return t_thrd.codegen_cxt.thr_codegen_obj != NULL && !t_thrd.codegen_cxt.g_runningInFmgr; + return t_thrd.codegen_cxt.thr_codegen_obj != NULL && !t_thrd.codegen_cxt.g_runningInFmgr + && !((dorado::GsCodeGen*)t_thrd.codegen_cxt.thr_codegen_obj)->IsCompiled(); } /** diff --git a/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.cpp b/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.cpp index 80c70babf..47ba67c16 100644 --- a/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.cpp +++ b/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.cpp @@ -23,11 +23,10 @@ */ #include "occ_transaction_manager.h" -#include "../utils/utilities.h" +#include "utilities.h" #include "cycles.h" #include "mot_engine.h" #include "row.h" -#include "row_header.h" #include "txn.h" #include "txn_access.h" #include "checkpoint_manager.h" @@ -42,82 +41,151 @@ OccTransactionManager::OccTransactionManager() : m_txnCounter(0), m_abortsCounter(0), m_writeSetSize(0), - m_rowsSetSize(0), - m_deleteSetSize(0), m_insertSetSize(0), m_dynamicSleep(100), m_rowsLocked(false), m_preAbort(true), - m_validationNoWait(true) + m_validationNoWait(true), + m_isTransactionCommited(false) {} OccTransactionManager::~OccTransactionManager() {} -bool OccTransactionManager::Init() +bool OccTransactionManager::PreAbortCheck(TxnManager* txMan, GcMaintenanceInfo& gcMemoryReserve) { - bool result = true; - return result; + TxnAccess* tx = txMan->m_accessMgr; + TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); + auto itr = orderedSet.begin(); + while (itr != orderedSet.end()) { + Access* ac = (*itr).second; + switch (ac->m_type) { + case WR: + m_writeSetSize++; + gcMemoryReserve.m_version_queue++; + break; + case DEL: + m_writeSetSize++; + if (ac->m_params.IsPrimarySentinel()) { + gcMemoryReserve.m_delete_queue++; + } else { + if (ac->m_params.IsIndexUpdate()) { + gcMemoryReserve.m_update_column_queue++; + } + } + gcMemoryReserve.m_generic_queue++; + break; + case INS: + m_insertSetSize++; + m_writeSetSize++; + if (ac->m_params.IsUpgradeInsert()) { + gcMemoryReserve.m_version_queue++; + } + break; + case RD_FOR_UPDATE: + case RD: + itr = orderedSet.erase(itr); + txMan->m_accessMgr->PubReleaseAccess(ac); + continue; + default: + break; + } + + if (m_preAbort) { + if (!QuickHeaderValidation(ac)) { + if (MOTEngine::GetInstance()->IsRecovering() && ResolveRecoveryOccConflict(txMan, ac) == RC_OK) { + (void)++itr; + continue; + } + return false; + } + } + (void)++itr; + } + return true; } -bool OccTransactionManager::CheckVersion(const Access* access) +bool OccTransactionManager::QuickVersionCheck(const Access* access) { - // We always validate on committed rows! - const Row* row = access->GetRowFromHeader(); - return (row->m_rowHeader.GetCSN() == access->m_tid); + if (access->m_params.IsPrimarySentinel()) { + // For Upgrade IOD - Verify the sentinel snapshot is still valid + // Check if the key is still visible + if (access->m_snapshot <= access->m_origSentinel->GetStartCSN()) { + return false; + } + MOT_ASSERT(access->m_csn == access->m_globalRow->GetCommitSequenceNumber()); + return (access->m_csn == access->m_origSentinel->GetData()->GetCommitSequenceNumber()); + } else { + if (access->m_params.IsSecondaryUniqueSentinel()) { + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (node->GetEndCSN() != Sentinel::SENTINEL_INIT_CSN) { + return false; + } + return (access->m_secondaryUniqueNode == node); + } else { + MOT_ASSERT(access->GetType() == AccessType::DEL); + // Check that the Sentinel is not deleted! + return (static_cast(access->GetSentinel())->GetEndCSN() == Sentinel::SENTINEL_INIT_CSN); + } + } +} + +bool OccTransactionManager::QuickInsertCheck(const Access* access) +{ + // Lets verify the inserts + Sentinel* sent = access->m_origSentinel; + if (access->m_params.IsUpgradeInsert() == false) { + // if the sent is committed we abort! + if (sent->IsCommited()) { + return false; + } + } else { + if (access->m_params.IsPrimarySentinel()) { + // For Upgrade IOD - Verify the sentinel snapshot is still valid + if (access->m_params.IsInsertOnDeletedRow()) { + if (access->m_origSentinel->GetData()->IsRowDeleted() == false) { + return false; + } + } + return (access->m_csn == access->m_origSentinel->GetData()->GetCommitSequenceNumber()); + } else { + if (access->m_params.IsSecondaryUniqueSentinel()) { + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (access->m_params.IsInsertOnDeletedRow()) { + // Check if node is deleted + if (node->GetEndCSN() == Sentinel::SENTINEL_INIT_CSN) { + return false; + } + } else { + // Check if node is committed + if (node->GetEndCSN() != Sentinel::SENTINEL_INIT_CSN) { + return false; + } + } + return (access->m_secondaryUniqueNode == node); + } else { + MOT_ASSERT(access->GetType() == AccessType::INS); + MOT_ASSERT(access->m_params.IsIndexUpdate() == true); + if (static_cast(access->GetSentinel())->GetEndCSN() == + Sentinel::SENTINEL_INIT_CSN) { + return false; + } + return (static_cast(access->GetSentinel())->GetStartCSN() == access->m_csn); + } + } + } + + return true; } bool OccTransactionManager::QuickHeaderValidation(const Access* access) { if (access->m_type != INS) { // For WR/DEL/RD_FOR_UPDATE lets verify CSN - return CheckVersion(access); + return QuickVersionCheck(access); } else { - // Lets verify the inserts - // For upgrade we verify the row - // csn has not changed! - Sentinel* sent = access->m_origSentinel; - if (access->m_params.IsUpgradeInsert()) { - if (access->m_params.IsDummyDeletedRow()) { - // Check is sentinel is deleted and CSN is VALID - ABA problem - if (sent->IsCommited() == false) { - if (sent->GetData()->GetCommitSequenceNumber() != access->m_tid) { - return false; - } - } else { - return false; - } - } else { - // We deleted internally!, we only need to check version - if (sent->GetData()->GetCommitSequenceNumber() != access->m_tid) { - return false; - } - } - } else { - // If the sent is committed or inserted-deleted we abort! - if (sent->IsCommited() or sent->GetData() != nullptr) { - return false; - } - } + return QuickInsertCheck(access); } - - return true; -} - -bool OccTransactionManager::ValidateReadSet(TxnManager* txMan) -{ - TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); - for (const auto& raPair : orderedSet) { - const Access* ac = raPair.second; - if (ac->m_type != RD) { - continue; - } - if (!ac->GetRowFromHeader()->m_rowHeader.ValidateRead(ac->m_tid)) { - return false; - } - } - - return true; } bool OccTransactionManager::ValidateWriteSet(TxnManager* txMan) @@ -125,10 +193,6 @@ bool OccTransactionManager::ValidateWriteSet(TxnManager* txMan) TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); for (const auto& raPair : orderedSet) { const Access* ac = raPair.second; - if (ac->m_type == RD) { - continue; - } - if (!QuickHeaderValidation(ac)) { return false; } @@ -136,106 +200,65 @@ bool OccTransactionManager::ValidateWriteSet(TxnManager* txMan) return true; } -RC OccTransactionManager::LockRows(TxnManager* txMan, uint32_t& numRowsLock) +RC OccTransactionManager::LockHeaders(TxnManager* txMan, uint32_t& numSentinelsLock) { RC rc = RC_OK; - TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); - numRowsLock = 0; - for (const auto& raPair : orderedSet) { - const Access* ac = raPair.second; - if (ac->m_type == RD) { - continue; - } - if (ac->m_params.IsPrimarySentinel()) { - Row* row = ac->GetRowFromHeader(); - row->m_rowHeader.Lock(); - numRowsLock++; - MOT_ASSERT(row->GetPrimarySentinel()->IsLocked() == true); - } - } - - return rc; -} - -bool OccTransactionManager::LockHeadersNoWait(TxnManager* txMan, uint32_t& numSentinelsLock) -{ uint64_t sleepTime = 1; uint64_t thdId = txMan->GetThdId(); TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); numSentinelsLock = 0; - while (numSentinelsLock != m_writeSetSize) { - for (const auto& raPair : orderedSet) { - const Access* ac = raPair.second; - if (ac->m_type == RD) { - continue; + if (m_validationNoWait) { + while (numSentinelsLock != m_writeSetSize) { + for (const auto& raPair : orderedSet) { + const Access* ac = raPair.second; + Sentinel* sent = ac->m_origSentinel; + if (!sent->TryLock(thdId)) { + break; + } + numSentinelsLock++; + // New insert row is already committed! + // Check if row has changed in sentinel + if (!QuickHeaderValidation(ac)) { + rc = RC_ABORT; + goto final; + } } - Sentinel* sent = ac->m_origSentinel; - if (!sent->TryLock(thdId)) { - break; - } - numSentinelsLock++; - if (ac->m_params.IsPrimaryUpgrade()) { - ac->m_auxRow->m_rowHeader.Lock(); - } - // New insert row is already committed! - // Check if row has changed in sentinel - if (!QuickHeaderValidation(ac)) { - return false; - } - } - if (numSentinelsLock != m_writeSetSize) { - ReleaseHeaderLocks(txMan, numSentinelsLock); - numSentinelsLock = 0; - if (m_preAbort) { - for (const auto& acPair : orderedSet) { - const Access* ac = acPair.second; - if (!QuickHeaderValidation(ac)) { - return false; + if (numSentinelsLock != m_writeSetSize) { + ReleaseHeaderLocks(txMan, numSentinelsLock); + numSentinelsLock = 0; + if (m_preAbort) { + for (const auto& acPair : orderedSet) { + const Access* ac = acPair.second; + if (!QuickHeaderValidation(ac)) { + return RC_ABORT; + } } } - } - if (sleepTime > LOCK_TIME_OUT) { - return false; - } else { - if (IsHighContention() == false) { - CpuCyclesLevelTime::Sleep(5); + if (!MOTEngine::GetInstance()->IsRecovering()) { + if (sleepTime > LOCK_TIME_OUT) { + return RC_ABORT; + } else { + if (!IsHighContention()) { + CpuCyclesLevelTime::Sleep(5); + } else { + (void)usleep(m_dynamicSleep); + } + sleepTime = sleepTime << 1; + } } else { - usleep(m_dynamicSleep); + (void)usleep(1000); } - sleepTime = sleepTime << 1; } } - } - - return true; -} - -RC OccTransactionManager::LockHeaders(TxnManager* txMan, uint32_t& numSentinelsLock) -{ - RC rc = RC_OK; - uint64_t thdId = txMan->GetThdId(); - TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); - numSentinelsLock = 0; - if (m_validationNoWait) { - if (!LockHeadersNoWait(txMan, numSentinelsLock)) { - rc = RC_ABORT; - goto final; - } } else { for (const auto& raPair : orderedSet) { const Access* ac = raPair.second; - if (ac->m_type == RD) { - continue; - } Sentinel* sent = ac->m_origSentinel; sent->Lock(thdId); numSentinelsLock++; - if (ac->m_params.IsPrimaryUpgrade()) { - ac->m_auxRow->m_rowHeader.Lock(); - } // New insert row is already committed! - // Check if row has chained in sentinel + // Check if row has changed in sentinel if (!QuickHeaderValidation(ac)) { rc = RC_ABORT; goto final; @@ -249,18 +272,15 @@ final: bool OccTransactionManager::PreAllocStableRow(TxnManager* txMan) { if (GetGlobalConfiguration().m_enableCheckpoint) { - GetCheckpointManager()->BeginCommit(txMan); - TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); for (const auto& raPair : orderedSet) { const Access* access = raPair.second; - if (access->m_type == RD) { + if (access->m_type == RD || (access->m_type == INS && access->m_params.IsUpgradeInsert() == false)) { continue; } if (access->m_params.IsPrimarySentinel()) { if (!GetCheckpointManager()->PreAllocStableRow(txMan, access->GetRowFromHeader(), access->m_type)) { GetCheckpointManager()->FreePreAllocStableRows(txMan); - GetCheckpointManager()->EndCommit(txMan); return false; } } @@ -269,108 +289,98 @@ bool OccTransactionManager::PreAllocStableRow(TxnManager* txMan) return true; } -bool OccTransactionManager::QuickVersionCheck(TxnManager* txMan, uint32_t& readSetSize) +bool OccTransactionManager::ReserveGcMemory(TxnManager* txMan, const GcMaintenanceInfo& gcMemoryReserve) { - int isolationLevel = txMan->GetTxnIsoLevel(); - TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); - readSetSize = 0; - for (const auto& raPair : orderedSet) { - const Access* ac = raPair.second; - if (ac->m_params.IsPrimarySentinel()) { - m_rowsSetSize++; - } - switch (ac->m_type) { - case RD_FOR_UPDATE: - case WR: - m_writeSetSize++; - break; - case DEL: - m_writeSetSize++; - m_deleteSetSize++; - break; - case INS: - m_insertSetSize++; - m_writeSetSize++; - break; - case RD: - if (isolationLevel > READ_COMMITED) { - readSetSize++; - } else { - continue; - } - break; - default: - break; - } - - if (m_preAbort) { - if (!QuickHeaderValidation(ac)) { - return false; - } - } + bool res = true; + GcManager* gc_manager = txMan->GetGcSession(); + MOT_ASSERT(gc_manager != nullptr); + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::DELETE_QUEUE, gcMemoryReserve.m_delete_queue); + if (!res) { + return false; + } + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::VERSION_QUEUE, gcMemoryReserve.m_version_queue); + if (!res) { + return false; + } + res = + gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::UPDATE_COLUMN_QUEUE, gcMemoryReserve.m_update_column_queue); + if (!res) { + return false; + } + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::GENERIC_QUEUE, gcMemoryReserve.m_generic_queue, true); + if (!res) { + return false; } - return true; -} + return res; +} +/* function name:ValidateOcc + function purpose:Verify that all operations done by the transaction are legal before the transaction is committed, + that is, they do not conflict with other concurrent transactions + input:transaction manager + output:Validation results(enum variable). + note:none + annotator:liushifa + annotate time:2023/08/25 10:59:07 + contact:3325287047@qq.com +*/ RC OccTransactionManager::ValidateOcc(TxnManager* txMan) -{ +{ // Initialize the number of locked sentinels to 0 uint32_t numSentinelLock = 0; m_rowsLocked = false; - TxnAccess* tx = txMan->m_accessMgr.Get(); + // Get the access manager for the transaction + TxnAccess* txnAccess = txMan->m_accessMgr; RC rc = RC_OK; - const uint32_t rowCount = tx->m_rowCnt; + const uint32_t rowCount = txnAccess->Size(); m_writeSetSize = 0; - m_rowsSetSize = 0; - m_deleteSetSize = 0; m_insertSetSize = 0; m_txnCounter++; - +// If no rows are accessed, the transaction is read-only and returns success directly if (rowCount == 0) { // READONLY return rc; } +// Initialize garbage collection maintenance information + GcMaintenanceInfo gcMemoryReserve{}; - uint32_t readSetSize = 0; - TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); - MOT_ASSERT(rowCount == orderedSet.size()); + MOT_ASSERT(rowCount == txnAccess->GetOrderedRowSet().size()); - /* Perform Quick Version check */ - if (!QuickVersionCheck(txMan, readSetSize)) { - rc = RC_ABORT; - goto final; - } - - MOT_LOG_DEBUG("Validate OCC rowCnt=%u RD=%u WR=%u\n", tx->m_rowCnt, tx->m_rowCnt - m_writeSetSize, m_writeSetSize); - rc = LockHeaders(txMan, numSentinelLock); - if (rc != RC_OK) { - goto final; - } - - // Validate rows in the read set and write set - if (readSetSize > 0) { - if (!ValidateReadSet(txMan)) { + do { + /* 1.Perform pre-abort check and pre-processing */ + if (!PreAbortCheck(txMan, gcMemoryReserve)) { rc = RC_ABORT; - goto final; + break; + } + // try to lock all headers + rc = LockHeaders(txMan, numSentinelLock); + if (rc != RC_OK) { + break; + } + // verify write set + if (!ValidateWriteSet(txMan)) { + rc = RC_ABORT; + break; } - } - if (!ValidateWriteSet(txMan)) { - rc = RC_ABORT; - goto final; - } - - // Pre-allocate stable row according to the checkpoint state. - if (!PreAllocStableRow(txMan)) { - rc = RC_MEMORY_ALLOCATION_ERROR; - goto final; - } - -final: + // Pre-allocate stable row according to the checkpoint state. + if (!PreAllocStableRow(txMan)) { + rc = RC_MEMORY_ALLOCATION_ERROR; + break; + } + // Reserve memory for possible garbage collection operations + if (!ReserveGcMemory(txMan, gcMemoryReserve)) { + rc = RC_MEMORY_ALLOCATION_ERROR; + break; + } + } while (0); + // If all the above steps are successful if (likely(rc == RC_OK)) { MOT_ASSERT(numSentinelLock == m_writeSetSize); + // mark the row as locked m_rowsLocked = true; } else { + // Release all header locks ReleaseHeaderLocks(txMan, numSentinelLock); if (likely(rc == RC_ABORT)) { m_abortsCounter++; @@ -380,15 +390,85 @@ final: return rc; } -void OccTransactionManager::RollbackInserts(TxnManager* txMan) +RC OccTransactionManager::ResolveRecoveryOccConflict(TxnManager* txMan, Access* access) { - return txMan->UndoInserts(); + Row* row = nullptr; + RC rc = RC_ABORT; + uint64_t endCSN = static_cast(-1); + MOT_ASSERT(access->m_type == INS); + switch (access->m_origSentinel->GetIndexOrder()) { + case IndexOrder::INDEX_ORDER_PRIMARY: + // Check what is the current row the sentinel is pointing + row = access->m_origSentinel->GetData(); + if (row) { + // Row must be deleted with Smaller CSN + if (row->IsRowDeleted() == false) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + MOT_ASSERT(access->m_origSentinel->GetData()->GetCommitSequenceNumber() > access->m_csn); + // Reset the global version and set to insert on delete + access->m_params.SetUpgradeInsert(); + access->m_params.SetInsertOnDeletedRow(); + access->m_globalRow = row; + access->m_csn = row->GetCommitSequenceNumber(); + access->m_snapshot = static_cast(-1); + rc = RC_OK; + } else { + MOT_ASSERT(false); + return RC_ABORT; + } + break; + case IndexOrder::INDEX_ORDER_SECONDARY: + endCSN = static_cast(access->m_origSentinel)->GetEndCSN(); + if (txMan->GetCommitSequenceNumber() <= endCSN) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + if (endCSN == Sentinel::SENTINEL_INIT_CSN) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + access->m_params.SetUpgradeInsert(); + access->m_params.SetInsertOnDeletedRow(); + access->m_csn = static_cast(access->m_origSentinel)->GetStartCSN(); + rc = RC_OK; + break; + case IndexOrder::INDEX_ORDER_SECONDARY_UNIQUE: + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (node != nullptr) { + if (txMan->GetCommitSequenceNumber() <= node->GetEndCSN()) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + } + // Reset Visible node + access->m_params.SetUpgradeInsert(); + if (node->GetEndCSN() < Sentinel::SENTINEL_INIT_CSN) { + access->m_params.SetInsertOnDeletedRow(); + } else { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + access->m_secondaryUniqueNode = node; + rc = RC_OK; + break; + } + return rc; } - -void OccTransactionManager::ApplyWrite(TxnManager* txMan) +void OccTransactionManager::WriteChanges(TxnManager* txMan) { - if (GetGlobalConfiguration().m_enableCheckpoint) { - TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + if (m_writeSetSize == 0 && m_insertSetSize == 0) { + return; + } + + MOTConfiguration& cfg = GetGlobalConfiguration(); + uint64_t commit_csn = txMan->GetCommitSequenceNumber(); + uint64_t transaction_id = txMan->GetInternalTransactionId(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + + // Stable rows for checkpoint needs to be created (copied from original row) before modifying the global rows. + if (cfg.m_enableCheckpoint) { for (const auto& raPair : orderedSet) { const Access* access = raPair.second; if (access->m_type == RD) { @@ -397,33 +477,40 @@ void OccTransactionManager::ApplyWrite(TxnManager* txMan) if (access->m_params.IsPrimarySentinel()) { // Pass the actual global row (access->GetRowFromHeader()), so that the stable row will have the // same CSN, rowid, etc as the original row before the modifications are applied. - GetCheckpointManager()->ApplyWrite(txMan, access->GetRowFromHeader(), access->m_type); + GetCheckpointManager()->ApplyWrite(txMan, access->GetRowFromHeader(), access); } } } -} - -void OccTransactionManager::WriteChanges(TxnManager* txMan) -{ - if (m_writeSetSize == 0 && m_insertSetSize == 0) { - return; - } - - LockRows(txMan, m_rowsSetSize); - - // Stable rows for checkpoint needs to be created (copied from original row) before modifying the global rows. - ApplyWrite(txMan); - - TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); // Update CSN with all relevant information on global rows // For deletes invalidate sentinels - rows still locked! for (const auto& raPair : orderedSet) { - const Access* access = raPair.second; - access->GetRowFromHeader()->m_rowHeader.WriteChangesToRow(access, txMan->GetCommitSequenceNumber()); + Access* access = raPair.second; + access->WriteGlobalChanges(commit_csn, transaction_id); } - // Treat Inserts + WriteSentinelChanges(txMan); + + // For Recovery operation:Update transactionID + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD) { + continue; + } + if (access->m_params.IsPrimarySentinel()) { + static_cast(access->m_origSentinel)->SetTransactionId(transaction_id); + } + } + + m_isTransactionCommited = true; +} + +void OccTransactionManager::WriteSentinelChanges(TxnManager* txMan) +{ + uint64_t commit_csn = txMan->GetCommitSequenceNumber(); + S_SentinelNodePool* sentinelObjectPool = txMan->m_accessMgr->GetSentinelObjectPool(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + if (m_insertSetSize > 0) { for (const auto& raPair : orderedSet) { Access* access = raPair.second; @@ -434,92 +521,92 @@ void OccTransactionManager::WriteChanges(TxnManager* txMan) if (access->m_params.IsUpgradeInsert() == false) { if (access->m_params.IsPrimarySentinel()) { MOT_ASSERT(access->m_origSentinel->IsDirty() == true); + MOT_ASSERT(access->m_origSentinel->IsLocked() == true); // Connect row and sentinel, row is set to absent and locked - access->m_origSentinel->SetNextPtr(access->GetRowFromHeader()); + access->m_origSentinel->SetStartCSN(commit_csn); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); // Current state: row is set to absent,sentinel is locked and not dirty // Readers will not see the row + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); access->GetTxnRow()->GetTable()->UpdateRowCount(1); } else { - // We only set the in the secondary sentinel! - access->m_origSentinel->SetNextPtr(access->GetRowFromHeader()->GetPrimarySentinel()); + if (access->m_params.IsSecondaryUniqueSentinel() == false) { + // We only set the in the secondary sentinel! + access->m_origSentinel->SetStartCSN(commit_csn); + access->m_origSentinel->SetEndCSN(Sentinel::SENTINEL_INIT_CSN); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()->GetPrimarySentinel()); + } else { + access->m_origSentinel->SetStartCSN(commit_csn); + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + access->m_origSentinel->SetNextPtr(node); + } } + // Unset Dirty - still locked! + access->m_origSentinel->UnSetDirty(); } else { - MOT_ASSERT(access->m_params.IsUniqueIndex() == true); - // Rows are locked and marked as deleted - if (access->m_params.IsPrimarySentinel()) { - /* Switch the locked row's in the sentinel - * The old row is locked and marked deleted - * The new row is locked - * Save previous row in the access! - * We need it for the row release! - */ - Row* row = access->GetRowFromHeader(); - access->m_localInsertRow = row; - access->m_origSentinel->SetNextPtr(access->m_auxRow); - // Add row to GC! - txMan->GetGcSession()->GcRecordObject(row->GetTable()->GetPrimaryIndex()->GetIndexId(), - row, - nullptr, - Row::RowDtor, - ROW_SIZE_FROM_POOL(row->GetTable())); + if (access->m_params.IsInsertOnDeletedRow()) { + if (access->m_params.IsPrimarySentinel()) { + access->GetTxnRow()->GetTable()->UpdateRowCount(1); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + } else if (access->m_params.IsUniqueIndex() == true) { + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + auto oldNode = static_cast(access->m_origSentinel)->GetTopNode(); + node->SetNextVersion(oldNode); + access->m_origSentinel->SetNextPtr(node); + } else { + MOT_ASSERT(access->m_params.IsIndexUpdate()); + // Revalidate End CSN + static_cast(access->m_origSentinel)->SetEndCSN(Sentinel::SENTINEL_INIT_CSN); + } } else { - // Set Sentinel for - access->m_origSentinel->SetNextPtr(access->m_auxRow->GetPrimarySentinel()); - } - // upgrade should not change the reference count! - if (access->m_origSentinel->IsCommited()) { - access->m_origSentinel->SetUpgradeCounter(); + // We delete row internally and insert a new row + if (access->m_params.IsPrimarySentinel()) { + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + } else { + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + auto oldNode = static_cast(access->m_origSentinel)->GetTopNode(); + oldNode->SetEndCSN(commit_csn); + node->SetNextVersion(oldNode); + access->m_origSentinel->SetNextPtr(node); + } } } } } - - // Treat Inserts - if (m_insertSetSize > 0) { - for (const auto& raPair : orderedSet) { - const Access* access = raPair.second; - if (access->m_type != INS) { - continue; - } - access->m_origSentinel->UnSetDirty(); - } - } - - CleanRowsFromIndexes(txMan); } - -void OccTransactionManager::CleanRowsFromIndexes(TxnManager* txMan) -{ - if (m_deleteSetSize == 0) { - return; - } - - TxnAccess* tx = txMan->m_accessMgr.Get(); - TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); - uint32_t numOfDeletes = m_deleteSetSize; - // use local counter to optimize - for (const auto& raPair : orderedSet) { - const Access* access = raPair.second; - if (access->m_type == DEL) { - numOfDeletes--; - access->GetTxnRow()->GetTable()->UpdateRowCount(-1); - MOT_ASSERT(access->m_params.IsUpgradeInsert() == false); - // Use Txn Row as row may change INSERT after DELETE leaves residue - txMan->RemoveKeyFromIndex(access->GetTxnRow(), access->m_origSentinel); - } - if (!numOfDeletes) { - break; - } - } -} - +/* function name:ReleaseHeaderLocks + function purpose:Releases any row header locks acquired by the transaction during its execution. + input:transaction managers and number of locks + output:none + note:none + annotator:liushifa + annotate time:2023/08/25 11:19:14 + contact:3325287047@qq.com +*/ void OccTransactionManager::ReleaseHeaderLocks(TxnManager* txMan, uint32_t numOfLocks) { if (numOfLocks == 0) { return; } - TxnAccess* tx = txMan->m_accessMgr.Get(); + TxnAccess* tx = txMan->m_accessMgr; TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); // use local counter to optimize for (const auto& raPair : orderedSet) { @@ -528,41 +615,7 @@ void OccTransactionManager::ReleaseHeaderLocks(TxnManager* txMan, uint32_t numOf continue; } else { numOfLocks--; - access->m_origSentinel->Release(); - if (access->m_params.IsPrimaryUpgrade()) { - access->m_auxRow->m_rowHeader.Release(); - } - } - if (!numOfLocks) { - break; - } - } -} - -void OccTransactionManager::ReleaseRowsLocks(TxnManager* txMan, uint32_t numOfLocks) -{ - if (numOfLocks == 0) { - return; - } - - TxnAccess* tx = txMan->m_accessMgr.Get(); - TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); - - // use local counter to optimize - for (const auto& raPair : orderedSet) { - const Access* access = raPair.second; - if (access->m_type == RD) { - continue; - } - - if (access->m_params.IsPrimarySentinel()) { - numOfLocks--; - access->GetRowFromHeader()->m_rowHeader.Release(); - if (access->m_params.IsUpgradeInsert()) { - // This is the global row that we switched! - // Currently it's in the gc! - access->m_localInsertRow->m_rowHeader.Release(); - } + access->m_origSentinel->Unlock(); } if (!numOfLocks) { break; @@ -574,6 +627,6 @@ void OccTransactionManager::CleanUp() { m_writeSetSize = 0; m_insertSetSize = 0; - m_rowsSetSize = 0; + m_isTransactionCommited = false; } } // namespace MOT diff --git a/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.h b/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.h index 721dc8d38..438dd36ba 100644 --- a/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.h +++ b/src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.h @@ -33,6 +33,14 @@ class Access; class TxnManager; constexpr uint64_t LOCK_TIME_OUT = 1 << 16; + +struct GcMaintenanceInfo { + GcMaintenanceInfo() = default; + uint32_t m_version_queue = 0; + uint32_t m_delete_queue = 0; + uint32_t m_update_column_queue = 0; + uint32_t m_generic_queue = 0; +}; /** * @class OccTransactionManager * @brief Optimistic concurrency control implementation. @@ -45,11 +53,6 @@ public: /** @brief Destructor. */ ~OccTransactionManager(); - /** - * @brief Initialize _writeSet, _readSet and _insertSet array. - */ - bool Init(); - /** * @brief Sets or clears the pre-abort flag. * @detail Determines whether to call quickVersionCheck() during @@ -81,10 +84,6 @@ public: */ RC ValidateOcc(TxnManager* tx); - RC LockHeaders(TxnManager* txMan, uint32_t& numSentinelsLock); - - RC LockRows(TxnManager* txMan, uint32_t& numRowsLock); - /** * @brief Writes all the changes in the write set of a transaction and * release the locks associated with all the write access items. @@ -92,17 +91,25 @@ public: */ void WriteChanges(TxnManager* txMan); - /** @brief remove all deleted keys from the global indices */ - void CleanRowsFromIndexes(TxnManager* txMan); - - /** @brief Rollack insert-set due to an abort */ - void RollbackInserts(TxnManager* txMan); - + /** + * @brief Writes all the changes in the insert set of a transaction + * and release the locks associated with all the write access items. + * @param txMan The committing transaction. + */ + void WriteSentinelChanges(TxnManager* txMan); + /* function name:ReleaseLocks + function purpose:Release the locks associated with the transaction. This may be done after the transaction commits or aborts. + input:transaction manager + output:none + note:none + annotator:liushifa + annotate time:2023/08/23 23:46:41 + contact:3325287047@qq.com + */ void ReleaseLocks(TxnManager* txMan) { if (m_rowsLocked) { ReleaseHeaderLocks(txMan, m_writeSetSize); - ReleaseRowsLocks(txMan, m_rowsSetSize); m_rowsLocked = false; } } @@ -138,58 +145,73 @@ public: return false; } } + /* function name:IsTransactionCommited + function purpose:Indicates whether the transaction has committed + input:none + output:Boolean value + note:none + annotator:liushifa + annotate time:2023/08/23 23:48:34 + contact:3325287047@qq.com + */ + inline bool IsTransactionCommited() const + { + return m_isTransactionCommited; + } + /* function name:GetTxnCounter + function purpose:Returns the number of transactions processed. + input:none + output:the number of transactions processed. + note:none + annotator:liushifa + annotate time:2023/08/23 23:49:40 + contact:3325287047@qq.com + */ + inline uint64_t GetTxnCounter() const + { + return m_txnCounter; + } private: - /** - * @brief Checks whether the transaction identifier in the original row - * in the access matches the transaction identifier in the access object. - * @detail The check is faster than a cc validate, but it does not - * guarantee correctness. If the versions do not match, cc - * verification will always fail too. However, if the function returns - * true, cc may fail during the verification (i.e. it may produce - * false positive reports). - */ - bool CheckVersion(const Access* access); + /** @brief Perform validation of current access */ + bool QuickVersionCheck(const Access* access); - /** @brief Validate Header for insert */ + /** @brief Perform validation of current access */ + bool QuickInsertCheck(const Access* access); + + /** @brief Perform pre-processing and validation */ + bool PreAbortCheck(TxnManager* txMan, GcMaintenanceInfo& gcMemoryReserve); + + /** @brief Validate Header for insert */ bool QuickHeaderValidation(const Access* access); - bool QuickVersionCheck(TxnManager* txMan, uint32_t& readSetSize); - - bool LockHeadersNoWait(TxnManager* txMan, uint32_t& numSentinelsLock); + /** @brief Lock all keys */ + RC LockHeaders(TxnManager* txMan, uint32_t& numSentinelsLock); + /** @brief Release Header locks */ void ReleaseHeaderLocks(TxnManager* txMan, uint32_t numOfLocks); - /** @brief Release all the locked rows */ - void ReleaseRowsLocks(TxnManager* txMan, uint32_t numOfLocks); + /** For Recovery, takes care of IDI (Insert, Delete, Insert) use case */ + RC ResolveRecoveryOccConflict(TxnManager* txMan, Access* access); - /** @brief Validate the read set */ - bool ValidateReadSet(TxnManager* txMan); - - /** @brief Validate the write set */ + /** @brief validate the write set */ bool ValidateWriteSet(TxnManager* txMan); /** @brief Pre-allocates stable row according to the checkpoint state. */ bool PreAllocStableRow(TxnManager* txMan); - /** @brief Sets stable row according to the checkpoint state. */ - void ApplyWrite(TxnManager* txMan); + /** @brief Pre-allocates GC memory for reclamation. */ + bool ReserveGcMemory(TxnManager* txMan, const GcMaintenanceInfo& gcMemoryReserve); /** @var transaction counter */ - uint32_t m_txnCounter; + uint64_t m_txnCounter; /** @var aborts counter */ - uint32_t m_abortsCounter; + uint64_t m_abortsCounter; /** @var Write set size. */ uint32_t m_writeSetSize; - /** @var total number of rows */ - uint32_t m_rowsSetSize; - - /** @var Write set size. */ - uint32_t m_deleteSetSize; - /** @var Write set size. */ uint32_t m_insertSetSize; @@ -203,6 +225,9 @@ private: /** @var Validate-no-wait configuration. */ bool m_validationNoWait; + + /** @var flag indicating whether transaction committed */ + bool m_isTransactionCommited; }; } // namespace MOT diff --git a/src/gausskernel/storage/mot/core/concurrency_control/row_header.cpp b/src/gausskernel/storage/mot/core/concurrency_control/row_header.cpp index ee10b1664..b69c952ba 100644 --- a/src/gausskernel/storage/mot/core/concurrency_control/row_header.cpp +++ b/src/gausskernel/storage/mot/core/concurrency_control/row_header.cpp @@ -31,7 +31,7 @@ #include "cycles.h" #include "debug_utils.h" #include "mot_engine.h" - +//this is a test namespace MOT { DECLARE_LOGGER(RowHeader, ConcurrenyControl); @@ -89,7 +89,15 @@ bool RowHeader::ValidateWrite(TransactionId tid) const { return (tid == GetCSN()); } - +/* function name:ValidateRead + function purpose:Validates the read operation on a row by checking its lock status and transaction id. + input:tid The transaction id of the current operation + output: True if the row can be safely read, otherwise false. + note:none + annotator:liushifa + annotate time:2023/10/05 22:51:42 + contact:3325287047@qq.com +*/ bool RowHeader::ValidateRead(TransactionId tid) const { if (IsLocked() or (tid != GetCSN())) { diff --git a/src/gausskernel/storage/mot/core/infra/synchronization/affinity.cpp b/src/gausskernel/storage/mot/core/infra/synchronization/affinity.cpp index 5b1f944ac..498bf9f52 100644 --- a/src/gausskernel/storage/mot/core/infra/synchronization/affinity.cpp +++ b/src/gausskernel/storage/mot/core/infra/synchronization/affinity.cpp @@ -24,7 +24,7 @@ #include #include -#include +#include #include "affinity.h" #include "global.h" @@ -47,81 +47,81 @@ void Affinity::Configure(uint64_t numaNodes, uint64_t physicalCoresNuma, Affinit m_affinityMode = affinityMode; } -uint32_t Affinity::GetAffineProcessor(uint64_t threadId) const +int Affinity::GetAffineProcessor(uint64_t threadId) const { - uint32_t result = INVALID_CPU_ID; + int result = INVALID_CPU_ID; threadId = threadId % (m_numaNodes * m_physicalCoresNuma); switch (m_affinityMode) { case AffinityMode::FILL_SOCKET_FIRST: { - result = (uint32_t)GetGlobalConfiguration().GetMappedCore(threadId); + result = GetGlobalConfiguration().GetMappedCore(threadId); break; } case AffinityMode::EQUAL_PER_SOCKET: { - threadId = threadId % (m_numaNodes * m_physicalCoresNuma); - uint32_t numaId = threadId % m_numaNodes; - uint32_t localProc = threadId / m_numaNodes; - result = (uint32_t)(numaId * m_physicalCoresNuma + localProc); + int numaId = threadId % m_numaNodes; + int localProc = threadId / m_numaNodes; + result = GetGlobalConfiguration().GetCoreFromNumaNodeByIndex(numaId, localProc); break; } case AffinityMode::FILL_PHYSICAL_FIRST: - result = (uint32_t)GetGlobalConfiguration().GetCoreByConnidFP(threadId); + result = GetGlobalConfiguration().GetCoreByConnidFP((int)threadId); break; default: MOT_LOG_ERROR("%s: Invalid affinity configuration: %d", __func__, (int)m_affinityMode); - result = INVALID_CPU_ID; break; } return result; } - -uint32_t Affinity::GetAffineNuma(uint64_t threadId) const +/* function name:GetAffineNuma + function purpose:To get the NUMA (Non-Uniform Memory Access) node ID associated with the specified thread Id + input:threadId + output:NUMA ID，If the query fails, INVALID_NODE_ID is returned. + note:no + annotator:liushifa + annotate time:2023/09/11 17:01:32 + contact:3325287047@qq.com +*/ +int Affinity::GetAffineNuma(uint64_t threadId) const { - uint32_t result = INVALID_NODE_ID; - threadId = threadId % (m_numaNodes * m_physicalCoresNuma); - + int result = INVALID_NODE_ID; + // Determine how to obtain the NUMA node ID based on the current affinity mode switch (m_affinityMode) { - case AffinityMode::FILL_SOCKET_FIRST: { - result = (uint32_t)GetGlobalConfiguration().GetCpuNode(GetGlobalConfiguration().GetMappedCore(threadId)); - break; - } - + case AffinityMode::FILL_SOCKET_FIRST: case AffinityMode::EQUAL_PER_SOCKET: - result = (uint32_t)threadId % m_numaNodes; - break; - case AffinityMode::FILL_PHYSICAL_FIRST: { - uint64_t realCoreCount = 0; - if (GetGlobalConfiguration().IsHyperThread() == true) { - realCoreCount = m_physicalCoresNuma / 2; - } else { - realCoreCount = m_physicalCoresNuma; - } - threadId = threadId % (m_numaNodes * realCoreCount); - result = (uint32_t)(threadId / realCoreCount); + int coreId = GetAffineProcessor(threadId); + result = GetGlobalConfiguration().GetCpuNode(coreId); break; } default: MOT_LOG_ERROR("%s: Invalid affinity configuration: %d", __func__, (int)m_affinityMode); - result = (uint32_t)INVALID_NODE_ID; break; } return result; } - -bool Affinity::SetAffinity(uint64_t threadId, uint32_t* threadCore /* = nullptr */) const +/* function name:SetAffinity + function purpose:Set CPU affinity for threads + input:thread id + output:Whether the affinity is successfully set + note:no + annotator:liushifa + annotate time:2023/09/11 16:57:04 + contact:3325287047@qq.com +*/ +bool Affinity::SetAffinity(uint64_t threadId, int* threadCore /* = nullptr */) const { bool result = true; - uint32_t coreId = GetAffineProcessor(threadId); + int coreId = GetAffineProcessor(threadId); cpu_set_t mask; + // Clear the mask, used to represent the CPU collection CPU_ZERO(&mask); - GetGlobalConfiguration().SetMaskToAllCoresinNumaSocket(mask, coreId); + GetGlobalConfiguration().SetMaskToAllCoresinNumaSocketByCoreId(mask, coreId); pthread_t currentThread = pthread_self(); // The following call forces migration of the thread if it is @@ -141,6 +141,7 @@ bool Affinity::SetAffinity(uint64_t threadId, uint32_t* threadCore /* = nullptr MOT_LOG_TRACE("Set current thread %u affinity to core %u (socket %u)", (unsigned)threadId, (unsigned)coreId, + //Gets the NUMA (Non-Uniform Memory Access) node ID associated with the specified thread Id (unsigned)GetAffineNuma(threadId)); } @@ -155,7 +156,7 @@ bool Affinity::SetNodeAffinity(int nodeId) bool result = true; cpu_set_t mask; CPU_ZERO(&mask); - GetGlobalConfiguration().SetMaskToAllCoresinNumaSocket2(mask, nodeId); + GetGlobalConfiguration().SetMaskToAllCoresinNumaSocketByNodeId(mask, nodeId); pthread_t currentThread = pthread_self(); // The following call forces migration of the thread if it is incorrectly placed @@ -173,10 +174,10 @@ bool Affinity::SetNodeAffinity(int nodeId) return result; } -static const char* AFFINITY_FILL_SOCKET_FIRST_STR = "fill-socket-first"; -static const char* AFFINITY_EQUAL_PER_SOCKET_STR = "equal-per-socket"; -static const char* AFFINITY_FILL_PHYSICAL_FIRST_STR = "fill-physical-first"; -static const char* AFFINITY_NONE_STR = "none"; +static const char* const AFFINITY_FILL_SOCKET_FIRST_STR = "fill-socket-first"; +static const char* const AFFINITY_EQUAL_PER_SOCKET_STR = "equal-per-socket"; +static const char* const AFFINITY_FILL_PHYSICAL_FIRST_STR = "fill-physical-first"; +static const char* const AFFINITY_NONE_STR = "none"; extern AffinityMode AffinityModeFromString(const char* affinityModeStr) { @@ -212,4 +213,12 @@ extern const char* AffinityModeToString(AffinityMode affinityMode) return "N/A"; } } + +extern bool ValidateAffinityMode(const char* affinityModeStr) +{ + if (AffinityModeFromString(affinityModeStr) == AffinityMode::AFFINITY_INVALID) { + return false; + } + return true; +} } // namespace MOT diff --git a/src/gausskernel/storage/mot/core/infra/synchronization/cycles.cpp b/src/gausskernel/storage/mot/core/infra/synchronization/cycles.cpp index 0edd6c938..6d1a84529 100644 --- a/src/gausskernel/storage/mot/core/infra/synchronization/cycles.cpp +++ b/src/gausskernel/storage/mot/core/infra/synchronization/cycles.cpp @@ -36,6 +36,19 @@ DECLARE_LOGGER(CpuCyclesLevelTime, System) double CpuCyclesLevelTime::m_cyclesPerSecond = 1.0; static constexpr double EPSILON = 1.0E-06; +/* function name:CpuCyclesLevelTime + function purpose:Initializes the cycles per second measurement. + The function measures the CPU cycles per second in order to convert CPU cycle durations + to real time durations. The measurement is done by capturing the CPU cycle count at + two different time intervals and then computing the difference. The result is then + adjusted to compute cycles per second. + input:none + output:none + note:none + annotator:liushifa + annotate time:2023/10/05 22:52:55 + contact:3325287047@qq.com +*/ void CpuCyclesLevelTime::Init() { if (m_cyclesPerSecond > 1.0) { diff --git a/src/gausskernel/storage/mot/core/memory/mm_buffer_allocator.cpp b/src/gausskernel/storage/mot/core/memory/mm_buffer_allocator.cpp index f60c892ca..0afeab7e6 100644 --- a/src/gausskernel/storage/mot/core/memory/mm_buffer_allocator.cpp +++ b/src/gausskernel/storage/mot/core/memory/mm_buffer_allocator.cpp @@ -108,7 +108,15 @@ extern int MemBufferAllocatorInit(MemBufferAllocator* bufferAllocator, int node, return result; } - +/* function name: FreeToBufferHeap + function purpose:Frees a given object to a specified buffer heap. + input:object Pointer to the object to be freed.serData Pointer to the buffer heap where the object will be freed. + output:none + note:none + annotator:liushifa + annotate time:2023/10/05 22:52:42 + contact:3325287047@qq.com +*/ static void FreeToBufferHeap(void* object, void* userData) { MemBufferHeap* bufferHeap = (MemBufferHeap*)userData; diff --git a/src/gausskernel/storage/mot/core/memory/mm_buffer_chunk.cpp b/src/gausskernel/storage/mot/core/memory/mm_buffer_chunk.cpp index 8fd0a1902..74faf37f7 100644 --- a/src/gausskernel/storage/mot/core/memory/mm_buffer_chunk.cpp +++ b/src/gausskernel/storage/mot/core/memory/mm_buffer_chunk.cpp @@ -118,9 +118,20 @@ extern void MemBufferChunkInit(MemBufferChunkHeader* chunkHeader, int16_t node, bufferHeader->m_prev = nullptr; // MM_CHUNK_BUFFER_HEADER_AT(chunkHeader, bufferIndex + 1); } } - +/* function name:MemBufferChunkOnDoubleFree + function purpose:Handles a scenario where a buffer chunk is freed twice (double free error). + input: + @param chunkHeader Pointer to the header of the buffer chunk that is being double-freed. + @param bufferHeader Pointer to the header of the buffer that contains the chunk being double-freed. + output:none + note:none + annotator:liushifa + annotate time:2023/10/05 22:53:42 + contact:3325287047@qq.com +*/ extern void MemBufferChunkOnDoubleFree(MemBufferChunkHeader* chunkHeader, MemBufferHeader* bufferHeader) { + // Log a panic message indicating a double free scenario with details MOT_LOG_PANIC( "Double free of buffer header %p [@%u -->%p] in chunk %p (node: %d, buffer-size=%u KB, allocated=%u/%u)", bufferHeader, diff --git a/src/gausskernel/storage/mot/core/memory/mm_buffer_heap.cpp b/src/gausskernel/storage/mot/core/memory/mm_buffer_heap.cpp index b5edd7980..04855cc0d 100644 --- a/src/gausskernel/storage/mot/core/memory/mm_buffer_heap.cpp +++ b/src/gausskernel/storage/mot/core/memory/mm_buffer_heap.cpp @@ -65,7 +65,19 @@ static void AssertChunkRemoved(MemBufferHeap* bufferHeap, MemBufferChunkHeader* #define ASSERT_HEAP_VALID(X) #define ASSERT_CHUNK_REMOVED(X, Y) #endif - +/* function name:MemBufferHeapInit + function purpose:Initializes a memory buffer heap. + input: + @param bufferHeap Pointer to the buffer heap to initialize. + @param node Specifies the NUMA node in which the memory buffer heap will be initialized. + @param bufferClass Classification of buffer based on its size. + @param allocType Specifies the type of allocation mechanism to use. + output:return Zero on successful initialization, and a non-zero error code otherwise. + note:none + annotator:liushifa + annotate time:2023/10/05 22:55:42 + contact:3325287047@qq.com +*/ extern int MemBufferHeapInit(MemBufferHeap* bufferHeap, int node, MemBufferClass bufferClass, MemAllocType allocType) { int result = MemLockInitialize(&bufferHeap->m_lock); diff --git a/src/gausskernel/storage/mot/core/memory/mm_numa.cpp b/src/gausskernel/storage/mot/core/memory/mm_numa.cpp index 7635b173c..949e353fe 100644 --- a/src/gausskernel/storage/mot/core/memory/mm_numa.cpp +++ b/src/gausskernel/storage/mot/core/memory/mm_numa.cpp @@ -266,7 +266,20 @@ extern void MemNumaGetStats(MemNumaStats* stats) stats->m_peakLocalMemoryBytes[i] = MOT_ATOMIC_LOAD(peakLocalMemoryBytes[i]); } } - +/* function name: MemNumaFormatStats + function purpose:Formats the NUMA statistics into a string buffer based on the report mode specified. + input: + @param indent The amount of whitespace to add before the statistics for readability. + * @param name Name of the allocation for the report. + * @param stringBuffer Pointer to the string buffer to which the formatted stats will be appended. + * @param stats Pointer to the statistics to format. + * @param reportMode Determines the mode of the report - summary or detailed. + output:none + note:none + annotator:liushifa + annotate time:2023/10/05 22:59:42 + contact:3325287047@qq.com +*/ extern void MemNumaFormatStats(int indent, const char* name, StringBuffer* stringBuffer, MemNumaStats* stats, MemReportMode reportMode /* = MEM_REPORT_SUMMARY */) { diff --git a/src/gausskernel/storage/mot/core/memory/mm_session_api.cpp b/src/gausskernel/storage/mot/core/memory/mm_session_api.cpp index 3014d8f05..904d9307b 100644 --- a/src/gausskernel/storage/mot/core/memory/mm_session_api.cpp +++ b/src/gausskernel/storage/mot/core/memory/mm_session_api.cpp @@ -117,7 +117,15 @@ extern int MemSessionApiInit() } return result; } - +/* function name:MemSessionApiDestroy + function purpose:Destroys the memory session API, which primarily manages memory allocations for each session/thread. + input:none + output:none + note:none + annotator:liushifa + annotate time:2023/10/01 22:51:42 + contact:3325287047@qq.com +*/ extern void MemSessionApiDestroy() { MOT_LOG_TRACE("Destroying Session API"); diff --git a/src/gausskernel/storage/mot/core/memory/sys_numa_api.cpp b/src/gausskernel/storage/mot/core/memory/sys_numa_api.cpp index dbd4bc4e5..1c98c208c 100644 --- a/src/gausskernel/storage/mot/core/memory/sys_numa_api.cpp +++ b/src/gausskernel/storage/mot/core/memory/sys_numa_api.cpp @@ -32,15 +32,15 @@ #include "postgres.h" #include "knl/knl_thread.h" -#include -#include +#include +#include #include #include #include -#include +#include #include -#include -#include +#include +#include #include #include #include @@ -52,19 +52,19 @@ #endif /* Flags for mbind */ -#define MPOL_MF_STRICT (1 << 0) /* Verify existing pages in the mapping */ -#define MPOL_MF_MOVE (1 << 1) /* Move pages owned by this process to conform to mapping */ -#define MPOL_MF_MOVE_ALL (1 << 2) /* Move every page to conform to mapping */ +#define MPOL_MF_STRICT (1U) /* Verify existing pages in the mapping */ +#define MPOL_MF_MOVE (1U << 1) /* Move pages owned by this process to conform to mapping */ +#define MPOL_MF_MOVE_ALL (1U << 2) /* Move every page to conform to mapping */ // some required utility macros -#define ROUND_UP(x, y) (((x) + (y)-1) & ~((y)-1)) +#define ROUND_UP(x, y) (((x) + (y) - 1) & ~((y) - 1)) #define CPU_BYTES(x) (ROUND_UP(x, sizeof(long))) #define CPU_LONGS(x) (CPU_BYTES(x) / sizeof(long)) -#define HOW_MANY(x, y) (((x) + ((y)-1)) / (y)) -#define BITS_PER_LONG (8 * sizeof(unsigned long)) -#define BITS_PER_INT (8 * sizeof(unsigned int)) -#define LONGS_PER_BITS(n) HOW_MANY(n, BITS_PER_LONG) +#define HOW_MANY(x, y) (((x) + ((y) - 1)) / (y)) +#define BITS_PER_ULONG ((unsigned int)8 * sizeof(unsigned long)) +#define BITS_PER_UINT ((unsigned int)8 * sizeof(unsigned int)) +#define LONGS_PER_BITS(n) HOW_MANY(n, BITS_PER_ULONG) #define BYTES_PER_BITS(x) ((x + 7) / 8) // define maximum number of NUMA nodes @@ -93,16 +93,21 @@ typedef struct PACKED BitMask_ST { unsigned long m_maskp[0]; } BitMaskSt; -#define BITMASK_NBYTES(bmp) (LONGS_PER_BITS(bmp->m_size) * sizeof(unsigned long)) -#define BITMASK_GETBIT(bmp, n) \ - (((unsigned int)n < bmp->m_size) ? ((bmp->m_maskp[n / BITS_PER_LONG] >> (n % BITS_PER_LONG)) & 1) : 0) -#define BITMASK_SETBIT(bmp, n) \ - if ((unsigned int)n < bmp->m_size) { \ - bmp->m_maskp[n / BITS_PER_LONG] |= 1UL << (n % BITS_PER_LONG); \ +#define BITMASK_NBYTES(bmp) (LONGS_PER_BITS((bmp)->m_size) * sizeof(unsigned long)) + +#define BITMASK_GETBIT(bmp, n) \ + (((unsigned int)(n) < (bmp)->m_size) \ + ? (((bmp)->m_maskp[(unsigned int)(n) / BITS_PER_ULONG] >> ((unsigned int)(n) % BITS_PER_ULONG)) & 1) \ + : 0) + +#define BITMASK_SETBIT(bmp, n) \ + if ((unsigned int)(n) < (bmp)->m_size) { \ + (bmp)->m_maskp[(unsigned int)(n) / BITS_PER_ULONG] |= 1UL << ((unsigned int)(n) % BITS_PER_ULONG); \ } -#define BITMASK_CLEARBIT(bmp, n) \ - if ((unsigned int)n < bmp->m_size) { \ - bmp->m_maskp[n / BITS_PER_LONG] &= ~(1UL << (n % BITS_PER_LONG)); \ + +#define BITMASK_CLEARBIT(bmp, n) \ + if ((unsigned int)(n) < (bmp)->m_size) { \ + (bmp)->m_maskp[(unsigned int)(n) / BITS_PER_ULONG] &= ~(1UL << ((unsigned int)(n) % BITS_PER_ULONG)); \ } #define BITMASK_FREE(x) \ @@ -111,11 +116,11 @@ typedef struct PACKED BitMask_ST { x = nullptr; \ } -#define BITMASK_ONSTACK(name, x) \ - int sz = sizeof(BitMaskSt) + (LONGS_PER_BITS(x) * sizeof(unsigned long)); \ - char _bmp_buf[sz]; \ - BitMaskSt* name = (BitMaskSt*)_bmp_buf; \ - bzero(name, sz); \ +#define BITMASK_ONSTACK(name, x) \ + size_t sz = sizeof(BitMaskSt) + (LONGS_PER_BITS(x) * sizeof(unsigned long)); \ + char _bmp_buf[sz]; \ + BitMaskSt* name = (BitMaskSt*)_bmp_buf; \ + bzero(name, sz); \ name->m_size = x; // Report errors/warnings @@ -145,13 +150,13 @@ extern void MotSysNumaDumpMMapError( int errnum, void* address, size_t length, int prot, int flags, int fd, off_t offset); // Static variables -static const char* MASK_SIZE_FILE = "/proc/self/status"; -static const char* NODE_MASK_PREFIX = "Mems_allowed:\t"; +static const char* const MASK_SIZE_FILE = "/proc/self/status"; +static const char* const NODE_MASK_PREFIX = "Mems_allowed:\t"; static const int NODE_MASK_PREFIX_SIZE = strlen(NODE_MASK_PREFIX); -static const char* NODE_CPU_MAP_FILE = "/sys/devices/system/node/node%d/cpumap"; +static const char* const NODE_CPU_MAP_FILE = "/sys/devices/system/node/node%d/cpumap"; #define MOTBindPolicy t_thrd.mot_cxt.bindPolicy #define MOTMBindFlags t_thrd.mot_cxt.mbindFlags -static int g_nodeMaskSize = 0; +static unsigned int g_nodeMaskSize = 0; static int g_cpuMaskSize = 0; static int g_maxConfNode = -1; static int g_maxConfCpu = -1; @@ -163,13 +168,33 @@ static BitMaskSt* g_nodesBm = nullptr; static BitMaskSt* g_memNodeBm = nullptr; static BitMaskSt** g_nodeCpuBm = nullptr; +/* function name:MotSysNumaInit + function purpose:Initialization of numa system + input:none + output:none + note:no + annotator:liushifa + annotate time:2023/09/16 11:05:20 + contact:3325287047@qq.com +*/ void MotSysNumaInit() { + // Set the size of the node mask representing available NUMA nodes in the system MotSysNumaSetNodemaskSize(); + + // Set or retrieve the number of configured NUMA nodes in the system MotSysNumaSetConfiguredNodes(); + + // Set or retrieve the maximum number of CPUs in the NUMA system MotSysNumaSetNumaMaxCpu(); + + // Set or retrieve the number of configured CPUs in the system MotSysNumaSetConfiguredCpus(); + + // Apply constraints to tasks to ensure they run on the correct NUMA nodes and aren't migrated elsewhere MotSysNumaSetTaskConstraints(); + + // Initialize the CPU mask for each NUMA node, representing the CPUs present on each node MotSysNumaNodeCpuMaskInit(); } @@ -210,7 +235,9 @@ void* MotSysNumaAllocInterleaved(size_t size) g_allNodesBm->m_size + 1, MOTMBindFlags) != 0) { MotSysNumaReportError("mbind"); - munmap(mem, size); + if (munmap(mem, size) != 0) { + MotSysNumaReportError("munmap"); + } mem = nullptr; } } @@ -237,7 +264,9 @@ void* MotSysNumaAllocOnNode(size_t size, int node) bmp->m_size + 1, MOTMBindFlags) != 0) { MotSysNumaReportError("mbind"); - munmap(mem, size); + if (munmap(mem, size) != 0) { + MotSysNumaReportError("munmap"); + } mem = nullptr; } } @@ -256,7 +285,9 @@ void* MotSysNumaAllocLocal(size_t size) } else { if (syscall(__NR_mbind, (intptr_t)mem, size, MPOL_PREFERRED, nullptr, 0, MOTMBindFlags) != 0) { MotSysNumaReportError("mbind"); - munmap(mem, size); + if (munmap(mem, size) != 0) { + MotSysNumaReportError("munmap"); + } mem = nullptr; } } @@ -277,19 +308,19 @@ static void* MotSysNumaMapAligned(size_t size, size_t align) if (((uint64_t)mem) % align != 0) { // take the slow route - munmap(mem, size); + (void)munmap(mem, size); mem = MOTSysNumaMmap(0, size + align, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); if (mem == MAP_FAILED) { return mem; } else { uint64_t offset = ((uint64_t)mem) % align; if (offset == 0) { // aligned this time so we only need to unmap suffix of align bytes - munmap(((char*)mem) + size, align); + (void)munmap(((char*)mem) + size, align); } else { size_t leadingUseless = align - offset; // guaranteed to be non-negative - munmap(mem, leadingUseless); + (void)munmap(mem, leadingUseless); void* alignedMem = ((char*)mem) + leadingUseless; - munmap(((char*)alignedMem) + size, offset); + (void)munmap(((char*)alignedMem) + size, offset); mem = alignedMem; } } @@ -317,7 +348,9 @@ void* MotSysNumaAllocAlignedInterleaved(size_t size, size_t align) g_allNodesBm->m_size + 1, MOTMBindFlags) != 0) { MotSysNumaReportError("mbind"); - munmap(mem, size); + if (munmap(mem, size) != 0) { + MotSysNumaReportError("munmap"); + } mem = nullptr; } } @@ -345,7 +378,9 @@ void* MotSysNumaAllocAlignedOnNode(size_t size, size_t align, int node) bmp->m_size + 1, MOTMBindFlags) != 0) { MotSysNumaReportError("mbind"); - munmap(mem, size); + if (munmap(mem, size) != 0) { + MotSysNumaReportError("munmap"); + } mem = nullptr; } } @@ -365,7 +400,9 @@ void* MotSysNumaAllocAlignedLocal(size_t size, size_t align) } else { if (syscall(__NR_mbind, (intptr_t)mem, size, MPOL_PREFERRED, nullptr, 0, MOTMBindFlags) != 0) { MotSysNumaReportError("mbind"); - munmap(mem, size); + if (munmap(mem, size) != 0) { + MotSysNumaReportError("munmap"); + } mem = nullptr; } } @@ -374,7 +411,9 @@ void* MotSysNumaAllocAlignedLocal(size_t size, size_t align) void MotSysNumaFree(void* mem, size_t size) { - munmap(mem, size); + if (munmap(mem, size) != 0) { + MotSysNumaReportError("munmap"); + } } int MotSysNumaAvailable() @@ -420,18 +459,36 @@ int MotSysNumaGetNode(int cpu) errno = EINVAL; return -1; } - +/* function name:MotSysNumaSetBindPolicy + function purpose:Set the binding policy for memory allocation + input: + output:none + note:no + annotator:liushifa + annotate time:2023/09/16 15:18:03 + contact:3325287047@qq.com +*/ void MotSysNumaSetBindPolicy(int strict) { MOTBindPolicy = (strict ? MPOL_BIND : MPOL_PREFERRED); } +/* function name:MotSysNumaSetStrict + function purpose:Set the strictness flag for memory binding + input:flag + output:none + note:no + annotator:liushifa + annotate time:2023/09/16 15:18:45 + contact:3325287047@qq.com +*/ void MotSysNumaSetStrict(int flag) { - if (flag) + if (flag) { MOTMBindFlags |= MPOL_MF_STRICT; - else + } else { MOTMBindFlags &= ~MPOL_MF_STRICT; + } } void MotSysNumaSetPreferred(int node) @@ -448,19 +505,28 @@ void MotSysNumaSetPreferred(int node) } } } - +/* function name:MotSysNumaCpuAllowed + function purpose:Determine whether the cpu is available to use the numa node + input:id of cpu + output:yes or no + note:none + annotator:liushifa + annotate time:2023/09/20 16:44:09 + contact:3325287047@qq.com +*/ bool MotSysNumaCpuAllowed(int cpu) { + // Determine whether the given cpu number is greater than the preset maximum value if (cpu > g_cpuMaskSize) { errno = EINVAL; return false; } - + // Check whether the global CPU bitmask has been initialized if (g_allCpusBm == nullptr) { errno = EINVAL; return false; } - + // Use the BITMASK_GETBIT macro to check if the given cpu number is set to 1 in the bitmask if (BITMASK_GETBIT(g_allCpusBm, cpu) == 1) { return true; } @@ -490,7 +556,7 @@ static int MotSysNumaParseBitmap(char* line, BitMaskSt* mask) if (p > line && sizeof(unsigned long) == 8) { oldp--; - errno_t erc = memmove_s(p, oldp - p + 1, p + 1, oldp - p + 1); + errno_t erc = memmove_s(p, (oldp - p) + 1, p + 1, (oldp - p) + 1); securec_check(erc, "\0", "\0"); while (p > line && *p != ',') { --p; @@ -540,51 +606,64 @@ static void MotSysNumaSetConfiguredNodes() g_maxConfNode = node; } } - closedir(d); + (void)closedir(d); } } - +/* function name:MotSysNumaSetConfiguredNodes + function purpose:Set numa constraints for system tasks + input:none + output:none + note:no + annotator:liushifa + annotate time:2023/09/16 11:13:50 + contact:3325287047@qq.com +*/ static void MotSysNumaSetTaskConstraints() { int i; char* buffer = nullptr; size_t buflen = 0; FILE* f; - + // Allocate bitmask space to represent all CPUs and NUMA nodes g_allCpusBm = MotSysNumaBitmaskAlloc(g_cpuMaskSize); g_allNodesBm = MotSysNumaBitmaskAlloc(g_nodeMaskSize); - - if ((f = fopen(MASK_SIZE_FILE, "r")) == nullptr) + // Open the NUMA configuration file for reading + if ((f = fopen(MASK_SIZE_FILE, "r")) == nullptr) { return; - - while (getline(&buffer, &buflen, f) > 0) { - char* mask = strrchr(buffer, '\t') + 1; - - if (strncmp(buffer, "Cpus_allowed:", 13) == 0) - g_numProcCpu = MotSysNumaReadBitmask(mask, g_allCpusBm); - - if (strncmp(buffer, "Mems_allowed:", 13) == 0) - g_numProcNode = MotSysNumaReadBitmask(mask, g_allNodesBm); } - fclose(f); + // Read the NUMA configuration file line by line + while (getline(&buffer, &buflen, f) > 0) { + MOT_ASSERT(buffer != nullptr); + char* mask = strrchr(buffer, '\t') + 1; + // Identify and process line for CPU configuration + if (strncmp(buffer, "Cpus_allowed:", 13) == 0) { + g_numProcCpu = MotSysNumaReadBitmask(mask, g_allCpusBm); + } + // Identify and process line for NUMA node configuration + if (strncmp(buffer, "Mems_allowed:", 13) == 0) { + g_numProcNode = MotSysNumaReadBitmask(mask, g_allNodesBm); + } + } + // Close the file and free the buffer + (void)fclose(f); if (buffer != nullptr) { free(buffer); } - + // If no valid CPUs are set, set all available CPUs as valid if (g_numProcCpu <= 0) { for (i = 0; i <= g_maxConfCpu; i++) { BITMASK_SETBIT(g_allCpusBm, i); } g_numProcCpu = g_maxConfCpu + 1; } - + // Limit the number of CPUs and adjust the bitmask if (g_numProcCpu > g_maxConfCpu + 1) { g_numProcCpu = g_maxConfCpu + 1; for (i = g_maxConfCpu + 1; i < (int)g_allCpusBm->m_size; i++) { BITMASK_CLEARBIT(g_allCpusBm, i); } } - + // If no valid NUMA nodes are set, set all available NUMA nodes as valid if (g_numProcNode <= 0) { for (i = 0; i <= g_maxConfNode; i++) { BITMASK_SETBIT(g_allNodesBm, i); @@ -663,7 +742,7 @@ static unsigned int MotSysNumaBitmaskWeight(const BitMaskSt* bmp) static int MotSysNumaReadBitmask(char* s, BitMaskSt* bmp) { char* end = s; - unsigned tmplen = (bmp->m_size + BITS_PER_INT - 1) / BITS_PER_INT; + unsigned tmplen = (bmp->m_size + BITS_PER_UINT - 1) / BITS_PER_UINT; unsigned int tmp[tmplen]; unsigned int* start = tmp; unsigned int n = 0; @@ -696,7 +775,7 @@ static int MotSysNumaReadBitmask(char* s, BitMaskSt* bmp) while (n) { unsigned int w; unsigned long x = 0; - for (w = 0; n && w < BITS_PER_LONG; w += BITS_PER_INT) { + for (w = 0; n && w < BITS_PER_ULONG; w += BITS_PER_UINT) { x |= ((unsigned long)start[n-- - 1] << w); } @@ -708,7 +787,7 @@ static int MotSysNumaReadBitmask(char* s, BitMaskSt* bmp) static void MotSysNumaNodeCpuMaskCleanup(void) { if (g_nodeCpuBm != nullptr) { - for (int i = 0; i < g_nodeMaskSize; i++) { + for (unsigned int i = 0; i < g_nodeMaskSize; i++) { BITMASK_FREE(g_nodeCpuBm[i]); } free(g_nodeCpuBm); @@ -728,7 +807,7 @@ static void MotSysNumaNodeCpuMaskInit(void) g_nodeCpuBm = (BitMaskSt**)calloc(g_nodeMaskSize, sizeof(BitMaskSt*)); - for (int i = 0; i < g_nodeMaskSize; i++) { + for (unsigned int i = 0; i < g_nodeMaskSize; i++) { len = 0; FILE* f = nullptr; char* line = nullptr; @@ -762,7 +841,7 @@ static void MotSysNumaNodeCpuMaskInit(void) } if (f != nullptr) { - fclose(f); + (void)fclose(f); } } } @@ -782,7 +861,7 @@ static void MotSysNumaSetNodemaskSize() } if (line != nullptr) free(line); - fclose(fp); + (void)fclose(fp); } if (g_nodeMaskSize == 0) { diff --git a/src/gausskernel/storage/mot/core/storage/column.cpp b/src/gausskernel/storage/mot/core/storage/column.cpp index c22830e8c..5805d5e38 100644 --- a/src/gausskernel/storage/mot/core/storage/column.cpp +++ b/src/gausskernel/storage/mot/core/storage/column.cpp @@ -115,7 +115,15 @@ const char* Column::ColumnErrorMsg(RC err) return "Foreign table does not support this column definition"; } } - +/* function name:Column + function purpose:Default constructor for the Column class. Initializes all member variables to their default values. + input:none + output:none + note:none + annotator:liushifa + annotate time:2023/10/02 22:51:42 + contact:3325287047@qq.com +*/ Column::Column() { this->m_id = 0; diff --git a/src/gausskernel/storage/mot/core/storage/index/index_factory.cpp b/src/gausskernel/storage/mot/core/storage/index/index_factory.cpp index cd57385aa..e0f98d064 100644 --- a/src/gausskernel/storage/mot/core/storage/index/index_factory.cpp +++ b/src/gausskernel/storage/mot/core/storage/index/index_factory.cpp @@ -27,18 +27,32 @@ #include "utilities.h" namespace MOT { -DECLARE_LOGGER(IndexFactory, Storage) - IMPLEMENT_CLASS_LOGGER(IndexFactory, Storage); + +/* function name:CreateIndex + function purpose:Creates a new index based on the provided parameters. + input: + @param indexOrder The order in which the index should be created. + @param indexingMethod The method to be used for indexing. + @param flavor The flavor/variant of the indexing tree to be used. + output:Returns a pointer to the newly created Index, or nullptr if the index creation fails. + note:no + annotator:liushifa + annotate time:2023/08/22 11:42:16 + contact:3325287047@qq.com +*/ Index* IndexFactory::CreateIndex(IndexOrder indexOrder, IndexingMethod indexingMethod, IndexTreeFlavor flavor) { Index* newIx = nullptr; - + // Attempt to create a primary index based on the specified method and flavor. newIx = CreatePrimaryIndex(indexingMethod, flavor); + + // If index creation was successful, set the order of the index. if (newIx != nullptr) { newIx->SetOrder(indexOrder); } else { + // Log an error if index creation failed. MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Create Index", "Failed to create primary index"); } diff --git a/src/gausskernel/storage/mot/core/system/mot_configuration.cpp b/src/gausskernel/storage/mot/core/system/mot_configuration.cpp index 2c5fea7bd..80cff10a5 100644 --- a/src/gausskernel/storage/mot/core/system/mot_configuration.cpp +++ b/src/gausskernel/storage/mot/core/system/mot_configuration.cpp @@ -47,9 +47,6 @@ constexpr uint64_t MOTConfiguration::SCALE_SECONDS; constexpr bool MOTConfiguration::DEFAULT_ENABLE_REDO_LOG; constexpr LoggerType MOTConfiguration::DEFAULT_LOGGER_TYPE; constexpr RedoLogHandlerType MOTConfiguration::DEFAULT_REDO_LOG_HANDLER_TYPE; -constexpr uint32_t MOTConfiguration::DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT; -constexpr uint32_t MOTConfiguration::MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT; -constexpr uint32_t MOTConfiguration::MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT; constexpr bool MOTConfiguration::DEFAULT_ENABLE_GROUP_COMMIT; constexpr uint64_t MOTConfiguration::DEFAULT_GROUP_COMMIT_SIZE; constexpr uint64_t MOTConfiguration::MIN_GROUP_COMMIT_SIZE; @@ -74,6 +71,13 @@ constexpr uint32_t MOTConfiguration::DEFAULT_CHECKPOINT_RECOVERY_WORKERS; constexpr uint32_t MOTConfiguration::MIN_CHECKPOINT_RECOVERY_WORKERS; constexpr uint32_t MOTConfiguration::MAX_CHECKPOINT_RECOVERY_WORKERS; constexpr bool MOTConfiguration::DEFAULT_ENABLE_LOG_RECOVERY_STATS; +constexpr RecoveryMode MOTConfiguration::DEFAULT_RECOVERY_MODE; +constexpr uint32_t MOTConfiguration::DEFAULT_PARALLEL_RECOVERY_WORKERS; +constexpr uint32_t MOTConfiguration::MIN_PARALLEL_RECOVERY_WORKERS; +constexpr uint32_t MOTConfiguration::MAX_PARALLEL_RECOVERY_WORKERS; +constexpr uint32_t MOTConfiguration::DEFAULT_PARALLEL_RECOVERY_QUEUE_SIZE; +constexpr uint32_t MOTConfiguration::MIN_PARALLEL_RECOVERY_QUEUE_SIZE; +constexpr uint32_t MOTConfiguration::MAX_PARALLEL_RECOVERY_QUEUE_SIZE; // machine configuration members constexpr uint16_t MOTConfiguration::DEFAULT_NUMA_NODES; constexpr uint16_t MOTConfiguration::DEFAULT_CORES_PER_CPU; @@ -169,11 +173,16 @@ constexpr uint64_t MOTConfiguration::MIN_GC_HIGH_RECLAIM_THRESHOLD_BYTES; constexpr uint64_t MOTConfiguration::MAX_GC_HIGH_RECLAIM_THRESHOLD_BYTES; // JIT configuration members constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN; -constexpr bool MOTConfiguration::DEFAULT_FORCE_MOT_PSEUDO_CODEGEN; +constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_QUERY_CODEGEN; +constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_SP_CODEGEN; +constexpr const char* MOTConfiguration::DEFAULT_MOT_SP_CODEGEN_ALLOWED; +constexpr const char* MOTConfiguration::DEFAULT_MOT_SP_CODEGEN_PROHIBITED; +constexpr const char* MOTConfiguration::DEFAULT_MOT_PURE_SP_CODEGEN; constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN_PRINT; constexpr uint32_t MOTConfiguration::DEFAULT_MOT_CODEGEN_LIMIT; constexpr uint32_t MOTConfiguration::MIN_MOT_CODEGEN_LIMIT; constexpr uint32_t MOTConfiguration::MAX_MOT_CODEGEN_LIMIT; +constexpr bool MOTConfiguration::DEFAULT_ENABLE_MOT_CODEGEN_PROFILE; // storage configuration constexpr bool MOTConfiguration::DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN; constexpr IndexTreeFlavor MOTConfiguration::DEFAULT_INDEX_TREE_FLAVOR; @@ -185,7 +194,8 @@ constexpr uint64_t MOTConfiguration::MAX_CFG_MONITOR_PERIOD_SECONDS; constexpr bool MOTConfiguration::DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION; constexpr uint64_t MOTConfiguration::DEFAULT_TOTAL_MEMORY_MB; -static constexpr unsigned int MAX_NUMA_NODES = 16u; +static constexpr int MAX_NUMA_NODES = 16; + #define IS_HYPER_THREAD_CMD "lscpu | grep \"Thread(s) per core:\" | awk '{print $4}'" static bool ParseLoggerType( @@ -256,6 +266,19 @@ static bool ParseRedoLogHandlerType(const std::string& cfgName, const std::strin return result; } +static bool ParseRecoveryMode(const std::string& cfgName, const std::string& variableName, const std::string& newValue, + RecoveryMode* variableValue) +{ + bool result = (cfgName == variableName); + if (result) { + *variableValue = RecoveryModeFromString(newValue.c_str()); + if (*variableValue == RecoveryMode::RECOVERY_INVALID) { + result = false; + } + } + return result; +} + static bool ParseBool( const std::string& cfgName, const std::string& variableName, const std::string& newValue, bool* variableValue) { @@ -305,7 +328,7 @@ static bool ParseUint16( { bool result = (cfgName == variableName); if (result) { - *variableValue = (uint16_t)std::stoul(newValue); + *variableValue = static_cast(std::stoul(newValue)); MOT_ASSERT(!newValue.empty()); } return result; @@ -365,8 +388,8 @@ bool MOTConfiguration::FindNumaNodes(int* maxNodes) return false; } - if ((unsigned int)*maxNodes >= MAX_NUMA_NODES) { - MOT_LOG_ERROR("sys_numa_num_configured_nodes() => %d, while MAX_NUMA_NODES=%u - cannot proceed", + if (*maxNodes >= MAX_NUMA_NODES) { + MOT_LOG_ERROR("sys_numa_num_configured_nodes() => %d, while MAX_NUMA_NODES=%d - cannot proceed", *maxNodes, MAX_NUMA_NODES); return false; @@ -374,9 +397,17 @@ bool MOTConfiguration::FindNumaNodes(int* maxNodes) return true; } - +/* function name:FindNumProcessors + function purpose:Discover the number of processors in the system and associate these processors with the numa nodes on which they reside. + input:CpuNodeMap，CpuMap.maxCoresPerNode + output:Whether the configuration is successful? + note:none + annotator:liushifa + annotate time:2023/09/20 17:33:02 + contact:3325287047@qq.com +*/ bool MOTConfiguration::FindNumProcessors(uint16_t* maxCoresPerNode, CpuNodeMap* cpuNodeMapper, CpuMap* cpuOsMapper) -{ +{ //Get the number of processors int nprocs = sysconf(_SC_NPROCESSORS_ONLN); if (nprocs <= 0) { MOT_LOG(LogLevel::LL_ERROR, "Invalid system configuration _SC_NPROCESSORS_ONLN=%d", nprocs); @@ -386,26 +417,27 @@ bool MOTConfiguration::FindNumProcessors(uint16_t* maxCoresPerNode, CpuNodeMap* uint16_t cpusPerNode[MAX_NUMA_NODES]; errno_t erc = memset_s(&(cpusPerNode[0]), sizeof(cpusPerNode), 0, sizeof(cpusPerNode)); securec_check(erc, "\0", "\0"); + //Iterate through each processor and associate its numa node for (int i = 0; i < nprocs; i++) { int node = MotSysNumaGetNode(i); if (node < 0) { MOT_LOG_ERROR("Invalid NUMA configuration numa_node_of_cpu(%d) => %d", i, node); return false; } - if ((unsigned int)node >= MAX_NUMA_NODES) { + if (node >= MAX_NUMA_NODES) { MOT_LOG_ERROR( - "CPU %d is located in node %d, while MAX_NUMA_NODES=%u - cannot proceed", i, node, MAX_NUMA_NODES); + "CPU %d is located in node %d, while MAX_NUMA_NODES=%d - cannot proceed", i, node, MAX_NUMA_NODES); return false; } cpusPerNode[node]++; (*cpuNodeMapper)[i] = node; - (*cpuOsMapper)[node].insert(i); + (void)(*cpuOsMapper)[node].insert(i); } // dynamically calculate number of CPU cores per NUMA node instead of hard-coded config value if (maxCoresPerNode != nullptr) { *maxCoresPerNode = 0; - for (unsigned int j = 0; j < MAX_NUMA_NODES; j++) { + for (int j = 0; j < MAX_NUMA_NODES; j++) { if (cpusPerNode[j] > *maxCoresPerNode) { *maxCoresPerNode = cpusPerNode[j]; } @@ -414,22 +446,34 @@ bool MOTConfiguration::FindNumProcessors(uint16_t* maxCoresPerNode, CpuNodeMap* return true; } +/* function name:SetMaskToAllCoresinNumaSocketByCoreId + function purpose:Find all the cores of the NUMA node where the given core ID is located,and update the mask to represent these cores + input:cpu_set, core id + output:none + note:none + annotator:liushifa + annotate time:2023/09/20 17:22:14 + contact:3325287047@qq.com +*/ -void MOTConfiguration::SetMaskToAllCoresinNumaSocket(cpu_set_t& mask, uint64_t threadId) +void MOTConfiguration::SetMaskToAllCoresinNumaSocketByCoreId(cpu_set_t& mask, int coreId) { - int nodeId = GetCpuNode(threadId); + // Get the NUMA node ID of the given core ID + int nodeId = GetCpuNode(coreId); + // Get all cores on the node from the map based on the node ID auto nodeMap = m_osCpuMap[nodeId]; - for (auto it = nodeMap.begin(); it != nodeMap.end(); ++it) { + for (auto it = nodeMap.begin(); it != nodeMap.end(); (void)++it) { + //Set the core in the CPU collection mask (that is, mark the core as available) if (MotSysNumaCpuAllowed(*it)) { CPU_SET(*it, &mask); } } } -void MOTConfiguration::SetMaskToAllCoresinNumaSocket2(cpu_set_t& mask, int nodeId) +void MOTConfiguration::SetMaskToAllCoresinNumaSocketByNodeId(cpu_set_t& mask, int nodeId) { auto nodeMap = m_osCpuMap[nodeId]; - for (auto it = nodeMap.begin(); it != nodeMap.end(); ++it) { + for (auto it = nodeMap.begin(); it != nodeMap.end(); (void)++it) { if (MotSysNumaCpuAllowed(*it)) { CPU_SET(*it, &mask); } @@ -440,7 +484,6 @@ MOTConfiguration::MOTConfiguration() : m_enableRedoLog(DEFAULT_ENABLE_REDO_LOG), m_loggerType(DEFAULT_LOGGER_TYPE), m_redoLogHandlerType(DEFAULT_REDO_LOG_HANDLER_TYPE), - m_asyncRedoLogBufferArrayCount(DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT), m_enableGroupCommit(DEFAULT_ENABLE_GROUP_COMMIT), m_groupCommitSize(DEFAULT_GROUP_COMMIT_SIZE), m_groupCommitTimeoutUSec(DEFAULT_GROUP_COMMIT_TIMEOUT_USEC), @@ -449,6 +492,9 @@ MOTConfiguration::MOTConfiguration() m_checkpointDir(DEFAULT_CHECKPOINT_DIR), m_checkpointSegThreshold(DEFAULT_CHECKPOINT_SEGSIZE_BYTES), m_checkpointWorkers(DEFAULT_CHECKPOINT_WORKERS), + m_recoveryMode(DEFAULT_RECOVERY_MODE), + m_parallelRecoveryWorkers(DEFAULT_PARALLEL_RECOVERY_WORKERS), + m_parallelRecoveryQueueSize(DEFAULT_PARALLEL_RECOVERY_QUEUE_SIZE), m_checkpointRecoveryWorkers(DEFAULT_CHECKPOINT_RECOVERY_WORKERS), m_abortBufferEnable(true), m_preAbort(true), @@ -492,18 +538,23 @@ MOTConfiguration::MOTConfiguration() m_sessionLargeBufferStoreSizeMB(DEFAULT_SESSION_LARGE_BUFFER_STORE_SIZE_MB), m_sessionLargeBufferStoreMaxObjectSizeMB(DEFAULT_SESSION_LARGE_BUFFER_STORE_MAX_OBJECT_SIZE_MB), m_sessionMaxHugeObjectSizeMB(DEFAULT_SESSION_MAX_HUGE_OBJECT_SIZE_MB), - m_gcEnable(DEFAULT_GC_ENABLE), m_gcReclaimThresholdBytes(DEFAULT_GC_RECLAIM_THRESHOLD_BYTES), m_gcReclaimBatchSize(DEFAULT_GC_RECLAIM_BATCH_SIZE), m_gcHighReclaimThresholdBytes(DEFAULT_GC_HIGH_RECLAIM_THRESHOLD_BYTES), m_enableCodegen(DEFAULT_ENABLE_MOT_CODEGEN), - m_forcePseudoCodegen(DEFAULT_FORCE_MOT_PSEUDO_CODEGEN), + m_enableQueryCodegen(DEFAULT_ENABLE_MOT_QUERY_CODEGEN), + m_enableSPCodegen(DEFAULT_ENABLE_MOT_SP_CODEGEN), + m_spCodegenAllowed(DEFAULT_MOT_SP_CODEGEN_ALLOWED), + m_spCodegenProhibited(DEFAULT_MOT_SP_CODEGEN_PROHIBITED), + m_pureSPCodegen(DEFAULT_MOT_PURE_SP_CODEGEN), m_enableCodegenPrint(DEFAULT_ENABLE_MOT_CODEGEN_PRINT), m_codegenLimit(DEFAULT_MOT_CODEGEN_LIMIT), + m_enableCodegenProfile(DEFAULT_ENABLE_MOT_CODEGEN_PROFILE), m_allowIndexOnNullableColumn(DEFAULT_ALLOW_INDEX_ON_NULLABLE_COLUMN), m_indexTreeFlavor(DEFAULT_INDEX_TREE_FLAVOR), m_configMonitorPeriodSeconds(DEFAULT_CFG_MONITOR_PERIOD_SECONDS), m_runInternalConsistencyValidation(DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION), + m_runInternalMvccConsistencyValidation(DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION), m_totalMemoryMb(DEFAULT_TOTAL_MEMORY_MB), m_suppressLog(0), m_loadExtraParams(false) @@ -516,7 +567,7 @@ void MOTConfiguration::Initialize() MotSysNumaInit(); int numa = DEFAULT_NUMA_NODES; if (FindNumaNodes(&numa)) { - m_numaNodes = (uint16_t)numa; + m_numaNodes = static_cast(numa); } else { MOT_LOG_WARN("Failed to infer the number of NUMA nodes on current machine, defaulting to %d", numa); m_numaAvailable = false; @@ -526,7 +577,7 @@ void MOTConfiguration::Initialize() if (FindNumProcessors(&cores, &m_cpuNodeMapper, &m_osCpuMap)) { m_coresPerCpu = cores; } else { - MOT_LOG_WARN("Failed to infer the number of cores on the current machine, defaulting to %u", (unsigned)cores); + MOT_LOG_WARN("Failed to infer the number of cores on the current machine, defaulting to %" PRIu16, cores); m_numaAvailable = false; } @@ -546,7 +597,6 @@ bool MOTConfiguration::SetFlag(const std::string& name, const std::string& value if (ParseBool(name, "enable_redo_log", value, &m_enableRedoLog)) { } else if (ParseLoggerType(name, "logger_type", value, &m_loggerType)) { } else if (ParseRedoLogHandlerType(name, "redo_log_handler_type", value, &m_redoLogHandlerType)) { - } else if (ParseUint32(name, "async_log_buffer_count", value, &m_asyncRedoLogBufferArrayCount)) { } else if (ParseBool(name, "enable_group_commit", value, &m_enableGroupCommit)) { } else if (ParseUint64(name, "group_commit_size", value, &m_groupCommitSize)) { } else if (ParseUint64(name, "group_commit_timeout_usec", value, &m_groupCommitTimeoutUSec)) { @@ -556,6 +606,9 @@ bool MOTConfiguration::SetFlag(const std::string& name, const std::string& value } else if (ParseUint64(name, "checkpoint_segsize", value, &m_checkpointSegThreshold)) { } else if (ParseUint32(name, "checkpoint_workers", value, &m_checkpointWorkers)) { } else if (ParseUint32(name, "checkpoint_recovery_workers", value, &m_checkpointRecoveryWorkers)) { + } else if (ParseRecoveryMode(name, "recovery_mode", value, &m_recoveryMode)) { + } else if (ParseUint32(name, "parallel_recovery_workers", value, &m_parallelRecoveryWorkers)) { + } else if (ParseUint32(name, "parallel_recovery_queue_size", value, &m_parallelRecoveryQueueSize)) { } else if (ParseBool(name, "abort_buffer_enable", value, &m_abortBufferEnable)) { } else if (ParseBool(name, "pre_abort", value, &m_preAbort)) { } else if (ParseValidation(name, "validation_lock", value, &m_validationLock)) { @@ -598,9 +651,11 @@ bool MOTConfiguration::SetFlag(const std::string& name, const std::string& value &m_sessionLargeBufferStoreMaxObjectSizeMB)) { } else if (ParseUint64(name, "session_max_huge_object_size_mb", value, &m_sessionMaxHugeObjectSizeMB)) { } else if (ParseBool(name, "enable_mot_codegen", value, &m_enableCodegen)) { - } else if (ParseBool(name, "force_mot_pseudo_codegen", value, &m_forcePseudoCodegen)) { + } else if (ParseBool(name, "enable_mot_query_codegen", value, &m_enableQueryCodegen)) { + } else if (ParseBool(name, "enable_mot_sp_codegen", value, &m_enableSPCodegen)) { } else if (ParseBool(name, "enable_mot_codegen_print", value, &m_enableCodegenPrint)) { } else if (ParseUint32(name, "mot_codegen_limit", value, &m_codegenLimit)) { + } else if (ParseBool(name, "enable_mot_codegen_profile", value, &m_enableCodegenProfile)) { } else if (ParseBool(name, "allow_index_on_nullable_column", value, &m_allowIndexOnNullableColumn)) { } else if (ParseIndexTreeFlavor(name, "index_tree_flavor", value, &m_indexTreeFlavor)) { } else if (ParseUint64(name, "config_monitor_period_seconds", value, &m_configMonitorPeriodSeconds)) { @@ -625,18 +680,18 @@ int MOTConfiguration::GetCpuNode(int cpu) const return (itr != m_cpuNodeMapper.end()) ? itr->second : -1; } -uint16_t MOTConfiguration::GetCoreByConnidFP(uint16_t cpu) const +int MOTConfiguration::GetCoreByConnidFP(int cpu) const { - uint16_t numOfRealCores = (IsHyperThread() == true) ? m_coresPerCpu / 2 : m_coresPerCpu; // 2 is for HyperThread + int numOfRealCores = (IsHyperThread()) ? m_coresPerCpu / 2 : m_coresPerCpu; // 2 is for HyperThread // Lets pin physical first - // cpu is already modulu of(numaNodes*num_cores) - if (cpu < m_numaNodes * numOfRealCores) { - cpu = cpu % (m_numaNodes * numOfRealCores); - uint16_t coreIndex = cpu % numOfRealCores; - uint16_t numaId = cpu / numOfRealCores; + // cpu is already modulo of (numaNodes * num_cores) + if (cpu < ((int)m_numaNodes * numOfRealCores)) { + cpu = cpu % ((int)m_numaNodes * numOfRealCores); + int coreIndex = cpu % numOfRealCores; + int numaId = cpu / numOfRealCores; int counter = 0; auto coreSet = m_osCpuMap.find(numaId); - for (auto it = coreSet->second.begin(); it != coreSet->second.end(); ++it) { + for (auto it = coreSet->second.begin(); it != coreSet->second.end(); (void)++it) { if (counter == coreIndex) { return *it; } @@ -651,10 +706,9 @@ uint16_t MOTConfiguration::GetCoreByConnidFP(uint16_t cpu) const int MOTConfiguration::GetMappedCore(int logicId) const { - int counter = 0; - for (auto it = m_osCpuMap.begin(); it != m_osCpuMap.end(); ++it) { - for (auto it2 = (*it).second.begin(); it2 != (*it).second.end(); ++it2) { + for (auto it = m_osCpuMap.begin(); it != m_osCpuMap.end(); (void)++it) { + for (auto it2 = (*it).second.begin(); it2 != (*it).second.end(); (void)++it2) { if (counter == logicId) { return *it2; } @@ -664,6 +718,22 @@ int MOTConfiguration::GetMappedCore(int logicId) const return -1; } +int MOTConfiguration::GetCoreFromNumaNodeByIndex(int numaId, int logicId) const +{ + auto coreSet = m_osCpuMap.find(numaId); + if (coreSet != m_osCpuMap.end()) { + int counter = 0; + for (auto it = coreSet->second.begin(); it != coreSet->second.end(); (void)++it) { + if (counter == logicId) { + return *it; + } + counter++; + } + } + + return INVALID_CPU_ID; +} + #define UPDATE_BOOL_CFG(var, cfgPath, defaultValue) \ UpdateBoolConfigItem(var, cfg->GetConfigValue(cfgPath, defaultValue, m_suppressLog == 0), cfgPath) @@ -887,6 +957,9 @@ void MOTConfiguration::LoadConfig() MOT_LOG_TRACE("Loading main configuration"); const LayeredConfigTree* cfg = ConfigManager::GetInstance().GetLayeredConfigTree(); + // load component log levels so we can enable traces in this logger + UpdateComponentLogLevel(); + // logger configuration if (m_loadExtraParams) { UPDATE_BOOL_CFG(m_enableRedoLog, "enable_redo_log", DEFAULT_ENABLE_REDO_LOG); @@ -897,11 +970,6 @@ void MOTConfiguration::LoadConfig() // overridden by the external configuration loader GaussdbConfigLoader (so in effect whatever is defined in // mot.conf is discarded). See GaussdbConfigLoader::ConfigureRedoLogHandler() for more details. UPDATE_USER_CFG(m_redoLogHandlerType, "redo_log_handler_type", DEFAULT_REDO_LOG_HANDLER_TYPE); - UPDATE_INT_CFG(m_asyncRedoLogBufferArrayCount, - "async_log_buffer_count", - DEFAULT_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT, - MIN_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT, - MAX_ASYNC_REDO_LOG_BUFFER_ARRAY_COUNT); // commit configuration UPDATE_BOOL_CFG(m_enableGroupCommit, "enable_group_commit", DEFAULT_ENABLE_GROUP_COMMIT); @@ -922,7 +990,7 @@ void MOTConfiguration::LoadConfig() UPDATE_BOOL_CFG(m_enableCheckpoint, "enable_checkpoint", DEFAULT_ENABLE_CHECKPOINT); } - if (!m_enableCheckpoint && m_enableRedoLog) { + if (!m_enableCheckpoint && !m_enableRedoLog) { if (m_suppressLog == 0) { MOT_LOG_WARN("Disabling redo_log forcibly as the checkpoint is disabled"); } @@ -949,6 +1017,34 @@ void MOTConfiguration::LoadConfig() MIN_CHECKPOINT_RECOVERY_WORKERS, MAX_CHECKPOINT_RECOVERY_WORKERS); + if (m_loadExtraParams) { + UPDATE_USER_CFG(m_recoveryMode, "recovery_mode", DEFAULT_RECOVERY_MODE); + } + + UPDATE_INT_CFG(m_parallelRecoveryWorkers, + "parallel_recovery_workers", + DEFAULT_PARALLEL_RECOVERY_WORKERS, + MIN_PARALLEL_RECOVERY_WORKERS, + MAX_PARALLEL_RECOVERY_WORKERS); + + UPDATE_INT_CFG(m_parallelRecoveryQueueSize, + "parallel_recovery_queue_size", + DEFAULT_PARALLEL_RECOVERY_QUEUE_SIZE, + MIN_PARALLEL_RECOVERY_QUEUE_SIZE, + MAX_PARALLEL_RECOVERY_QUEUE_SIZE); + + if (m_parallelRecoveryQueueSize < m_parallelRecoveryWorkers) { + if (m_suppressLog == 0) { + MOT_LOG_WARN("Invalid recovery configuration: parallel_recovery_queue_size (%" PRIu32 ") is lesser than " + "parallel_recovery_workers (%" PRIu32 "), changing parallel_recovery_queue_size to (%" PRIu32 + ")", + m_parallelRecoveryQueueSize, + m_parallelRecoveryWorkers, + m_parallelRecoveryWorkers); + } + m_parallelRecoveryQueueSize = m_parallelRecoveryWorkers; // At least one transaction per processor + } + // Tx configuration - not configurable yet if (m_loadExtraParams) { UPDATE_BOOL_CFG(m_abortBufferEnable, "tx_abort_buffers_enable", true); @@ -983,7 +1079,7 @@ void MOTConfiguration::LoadConfig() // log configuration UPDATE_USER_CFG(m_logLevel, "log_level", DEFAULT_LOG_LEVEL); - SetGlobalLogLevel(m_logLevel); + (void)SetGlobalLogLevel(m_logLevel); if (m_loadExtraParams) { UPDATE_USER_CFG(m_numaErrorsLogLevel, "numa_errors_log_level", DEFAULT_NUMA_ERRORS_LOG_LEVEL); UPDATE_USER_CFG(m_numaWarningsLogLevel, "numa_warnings_log_level", DEFAULT_NUMA_WARNINGS_LOG_LEVEL); @@ -994,7 +1090,6 @@ void MOTConfiguration::LoadConfig() LoadMemConfig(); // GC configuration - UPDATE_BOOL_CFG(m_gcEnable, "enable_gc", DEFAULT_GC_ENABLE); UPDATE_ABS_MEM_CFG(m_gcReclaimThresholdBytes, "reclaim_threshold", DEFAULT_GC_RECLAIM_THRESHOLD, @@ -1015,10 +1110,19 @@ void MOTConfiguration::LoadConfig() // JIT configuration UPDATE_BOOL_CFG(m_enableCodegen, "enable_mot_codegen", DEFAULT_ENABLE_MOT_CODEGEN); - UPDATE_BOOL_CFG(m_forcePseudoCodegen, "force_mot_pseudo_codegen", DEFAULT_FORCE_MOT_PSEUDO_CODEGEN); + UPDATE_BOOL_CFG(m_enableQueryCodegen, "enable_mot_query_codegen", DEFAULT_ENABLE_MOT_QUERY_CODEGEN); + UPDATE_BOOL_CFG(m_enableSPCodegen, "enable_mot_sp_codegen", DEFAULT_ENABLE_MOT_SP_CODEGEN); + + if (m_loadExtraParams) { + UPDATE_STRING_CFG(m_spCodegenAllowed, "mot_sp_codegen_allowed", DEFAULT_MOT_SP_CODEGEN_ALLOWED); + UPDATE_STRING_CFG(m_spCodegenProhibited, "mot_sp_codegen_prohibited", DEFAULT_MOT_SP_CODEGEN_PROHIBITED); + UPDATE_STRING_CFG(m_pureSPCodegen, "mot_pure_sp_codegen", DEFAULT_MOT_PURE_SP_CODEGEN); + } + UPDATE_BOOL_CFG(m_enableCodegenPrint, "enable_mot_codegen_print", DEFAULT_ENABLE_MOT_CODEGEN_PRINT); UPDATE_INT_CFG( m_codegenLimit, "mot_codegen_limit", DEFAULT_MOT_CODEGEN_LIMIT, MIN_MOT_CODEGEN_LIMIT, MAX_MOT_CODEGEN_LIMIT); + UPDATE_BOOL_CFG(m_enableCodegenProfile, "enable_mot_codegen_profile", DEFAULT_ENABLE_MOT_CODEGEN_PROFILE); // storage configuration if (m_loadExtraParams) { @@ -1038,11 +1142,11 @@ void MOTConfiguration::LoadConfig() UPDATE_BOOL_CFG(m_runInternalConsistencyValidation, "internal_consistency_validation", DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION); + UPDATE_BOOL_CFG(m_runInternalMvccConsistencyValidation, + "internal_mvcc_consistency_validation", + DEFAULT_RUN_INTERNAL_CONSISTENCY_VALIDATION); } - // load component log levels - UpdateComponentLogLevel(); - MOT_LOG_TRACE("Main configuration loaded"); } @@ -1083,7 +1187,7 @@ void MOTConfiguration::UpdateMemConfigItem(uint64_t& oldValue, const char* name, UpdateIntConfigItem(oldValue, defaultValueBytes / scale, name, lowerBound, upperBound); } else { // now we carefully examine the value type - ConfigValue* cfgValue = (ConfigValue*)cfgItem; + const ConfigValue* cfgValue = (const ConfigValue*)cfgItem; if (cfgValue->IsIntegral()) { // only a number was specified, so it is interpreted as bytes uint64_t memoryValueBytes = @@ -1155,7 +1259,7 @@ void MOTConfiguration::UpdateTimeConfigItem(uint64_t& oldValue, const char* name UpdateIntConfigItem(oldValue, defaultValueUSecs / scale, name, lowerBound, upperBound); } else { // now we carefully examine the value type - ConfigValue* cfgValue = (ConfigValue*)cfgItem; + const ConfigValue* cfgValue = (const ConfigValue*)cfgItem; if (cfgValue->IsIntegral()) { // only a number was specified, so it is interpreted as micro-seconds uint64_t timeValueUSecs = cfg->GetIntegerConfigValue(name, defaultValueUSecs, m_suppressLog == 0); @@ -1212,7 +1316,7 @@ void MOTConfiguration::UpdateComponentLogLevel() componentName.c_str(), TypeFormatter::ToString(componentLevel, logLevelStr)); } - SetLogComponentLogLevel(componentName.c_str(), componentLevel); + (void)SetLogComponentLogLevel(componentName.c_str(), componentLevel); } // all configuration values are logger configuration pairs (loggerName=log_level) @@ -1238,7 +1342,7 @@ void MOTConfiguration::UpdateComponentLogLevel() componentName.c_str(), TypeFormatter::ToString(loggerLevel, logLevelStr)); } - SetLoggerLogLevel(componentName.c_str(), loggerName.c_str(), loggerLevel); + (void)SetLoggerLogLevel(componentName.c_str(), loggerName.c_str(), loggerLevel); } ++loggerItr; } @@ -1278,12 +1382,16 @@ int MOTConfiguration::ParseMemoryPercent(const char* memoryValue) MOT_LOG_WARN("Invalid memory value format: %s (percentage value not in the range [0, 100])", memoryValue); } else { mot_string extra(endptr + 1); - extra.trim(); - if (extra.length() != 0) { - MOT_LOG_WARN("Invalid memory value format: %s (trailing characters after %%)", memoryValue); + if (!extra.is_valid()) { + MOT_LOG_ERROR("Failed to allocate memory string: %s", endptr + 1); } else { - MOT_LOG_TRACE("Parsed percentage: %d%%", percent); - result = percent; + extra.trim(); + if (extra.length() != 0) { + MOT_LOG_WARN("Invalid memory value format: %s (trailing characters after %%)", memoryValue); + } else { + MOT_LOG_TRACE("Parsed percentage: %d%%", percent); + result = percent; + } } } return result; @@ -1314,7 +1422,7 @@ uint64_t MOTConfiguration::ParseMemoryPercentTotal(const char* memoryValue, uint memoryValueBytes = m_totalMemoryMb * MEGA_BYTE * percent / 100; if (m_suppressLog == 0) { MOT_LOG_INFO( - "Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, memoryValueBytes / 1024ul / 1024ul); + "Loaded %s: %d%% from total = %" PRIu64 " MB", cfgPath, percent, (memoryValueBytes / 1024ul) / 1024ul); } } else { MOT_LOG_WARN("Invalid %s memory format: illegal percent specification", cfgPath); @@ -1330,39 +1438,40 @@ uint64_t MOTConfiguration::ParseMemoryUnit(const char* memoryValue, uint64_t def if (endptr == memoryValue) { MOT_LOG_WARN("Invalid %s memory value format: %s (expecting value digits)", cfgPath, memoryValue); } else if (*endptr == 0) { - MOT_LOG_TRACE("Missing %s memory value units: bytes assumed", cfgPath, memoryValue); - memoryValueBytes = value; + MOT_LOG_TRACE("Missing %s memory value units: %s (kilobytes assumed)", cfgPath, memoryValue); + memoryValueBytes = ((uint64_t)value) * 1024; } else { // get unit type and convert to mega-bytes mot_string suffix(endptr); - suffix.trim(); - if (suffix.compare_no_case("TB") == 0) { - MOT_LOG_TRACE("Loaded %s: %u TB", cfgPath, value); - memoryValueBytes = ((uint64_t)value) * 1024ull * 1024ull * 1024ull * 1024ull; - } else if (suffix.compare_no_case("GB") == 0) { - MOT_LOG_TRACE("Loaded %s: %u GB", cfgPath, value); - memoryValueBytes = ((uint64_t)value) * 1024ull * 1024ull * 1024ull; - } else if (suffix.compare_no_case("MB") == 0) { - MOT_LOG_TRACE("Loaded %s: %u MB", cfgPath, value); - memoryValueBytes = ((uint64_t)value) * 1024ull * 1024ull; - } else if (suffix.compare_no_case("KB") == 0) { - MOT_LOG_TRACE("Loaded %s: %u KB", cfgPath, value); - memoryValueBytes = ((uint64_t)value) * 1024; - } else if (suffix.compare_no_case("bytes") == 0) { - MOT_LOG_TRACE("Loaded %s: %u bytes", cfgPath, value); - memoryValueBytes = value; + if (!suffix.is_valid()) { + MOT_LOG_ERROR("Failed to allocate memory string: %s", endptr); } else { - MOT_LOG_WARN("Invalid %s memory value format: %s (invalid unit specifier '%s' - should be one of TB, GB, " - "MB, KB or bytes)", - cfgPath, - memoryValue, - suffix.c_str()); + suffix.trim(); + if (suffix.compare_no_case("TB") == 0) { + MOT_LOG_TRACE("Loaded %s: %u TB", cfgPath, value); + memoryValueBytes = ((uint64_t)value) * 1024ull * 1024ull * 1024ull * 1024ull; + } else if (suffix.compare_no_case("GB") == 0) { + MOT_LOG_TRACE("Loaded %s: %u GB", cfgPath, value); + memoryValueBytes = ((uint64_t)value) * 1024ull * 1024ull * 1024ull; + } else if (suffix.compare_no_case("MB") == 0) { + MOT_LOG_TRACE("Loaded %s: %u MB", cfgPath, value); + memoryValueBytes = ((uint64_t)value) * 1024ull * 1024ull; + } else if (suffix.compare_no_case("KB") == 0) { + MOT_LOG_TRACE("Loaded %s: %u KB", cfgPath, value); + memoryValueBytes = ((uint64_t)value) * 1024; + } else { + MOT_LOG_WARN("Invalid %s memory value format: %s (invalid unit specifier '%s' - should be one of TB, " + "GB, MB or KB)", + cfgPath, + memoryValue, + suffix.c_str()); + } } } return memoryValueBytes; } -static inline bool IsTimeUnitDays(mot_string& suffix) +static inline bool IsTimeUnitDays(const mot_string& suffix) { if ((suffix.compare_no_case("d") == 0) || (suffix.compare_no_case("days") == 0) || (suffix.compare_no_case("day") == 0)) { @@ -1371,7 +1480,7 @@ static inline bool IsTimeUnitDays(mot_string& suffix) return false; } -static inline bool IsTimeUnitHours(mot_string& suffix) +static inline bool IsTimeUnitHours(const mot_string& suffix) { if ((suffix.compare_no_case("h") == 0) || (suffix.compare_no_case("hours") == 0) || (suffix.compare_no_case("hour") == 0)) { @@ -1380,7 +1489,7 @@ static inline bool IsTimeUnitHours(mot_string& suffix) return false; } -static inline bool IsTimeUnitMinutes(mot_string& suffix) +static inline bool IsTimeUnitMinutes(const mot_string& suffix) { if ((suffix.compare_no_case("m") == 0) || (suffix.compare_no_case("mins") == 0) || (suffix.compare_no_case("minutes") == 0) || (suffix.compare_no_case("min") == 0) || @@ -1390,7 +1499,7 @@ static inline bool IsTimeUnitMinutes(mot_string& suffix) return false; } -static inline bool IsTimeUnitSeconds(mot_string& suffix) +static inline bool IsTimeUnitSeconds(const mot_string& suffix) { if ((suffix.compare_no_case("s") == 0) || (suffix.compare_no_case("secs") == 0) || (suffix.compare_no_case("seconds") == 0) || (suffix.compare_no_case("sec") == 0) || @@ -1400,7 +1509,7 @@ static inline bool IsTimeUnitSeconds(mot_string& suffix) return false; } -static inline bool IsTimeUnitMilliSeconds(mot_string& suffix) +static inline bool IsTimeUnitMilliSeconds(const mot_string& suffix) { if ((suffix.compare_no_case("ms") == 0) || (suffix.compare_no_case("millis") == 0) || (suffix.compare_no_case("milliseconds") == 0) || (suffix.compare_no_case("milli") == 0) || @@ -1410,7 +1519,7 @@ static inline bool IsTimeUnitMilliSeconds(mot_string& suffix) return false; } -static inline bool IsTimeUnitMicroSeconds(mot_string& suffix) +static inline bool IsTimeUnitMicroSeconds(const mot_string& suffix) { if ((suffix.compare_no_case("us") == 0) || (suffix.compare_no_case("micros") == 0) || (suffix.compare_no_case("microseconds") == 0) || (suffix.compare_no_case("micro") == 0) || @@ -1428,35 +1537,37 @@ uint64_t MOTConfiguration::ParseTimeValueMicros(const char* timeValue, uint64_t if (endptr == timeValue) { MOT_LOG_WARN("Invalid %s time value format: %s (expecting value digits)", cfgPath, timeValue); } else if (*endptr == 0) { - MOT_LOG_WARN("Invalid %s time value format: %s (expecting unit type after value)", cfgPath, timeValue); + MOT_LOG_TRACE("Missing %s time value units: %s (milliseconds assumed)", cfgPath, timeValue); + timeValueMicros = ((uint64_t)value) * 1000ull; } else { // get unit type and convert to micro-seconds mot_string suffix(endptr); - suffix.trim(); - if (IsTimeUnitDays(suffix)) { - MOT_LOG_TRACE("Loaded %s: %u days", cfgPath, value); - timeValueMicros = ((uint64_t)value) * 24ull * 60ull * 60ull * 1000ull * 1000ull; - } else if (IsTimeUnitHours(suffix)) { - MOT_LOG_TRACE("Loaded %s: %u hours", cfgPath, value); - timeValueMicros = ((uint64_t)value) * 60ull * 60ull * 1000ull * 1000ull; - } else if (IsTimeUnitMinutes(suffix)) { - MOT_LOG_TRACE("Loaded %s: %u minutes", cfgPath, value); - timeValueMicros = ((uint64_t)value) * 60ull * 1000ull * 1000ull; - } else if (IsTimeUnitSeconds(suffix)) { - MOT_LOG_TRACE("Loaded %s: %u seconds", cfgPath, value); - timeValueMicros = ((uint64_t)value) * 1000ull * 1000ull; - } else if (IsTimeUnitMilliSeconds(suffix)) { - MOT_LOG_TRACE("Loaded %s: %u milli-seconds", cfgPath, value); - timeValueMicros = ((uint64_t)value) * 1000ull; - } else if (IsTimeUnitMicroSeconds(suffix)) { - MOT_LOG_TRACE("Loaded %s: %u micro-seconds", cfgPath, value); - timeValueMicros = ((uint64_t)value); + if (!suffix.is_valid()) { + MOT_LOG_ERROR("Failed to allocate time unit string: %s", endptr); } else { - MOT_LOG_WARN("Invalid %s time value format: %s (invalid unit specifier '%s' - should be one of d, h, m, s, " - "ms or us)", - cfgPath, - timeValue, - suffix.c_str()); + suffix.trim(); + if (IsTimeUnitDays(suffix)) { + MOT_LOG_TRACE("Loaded %s: %u days", cfgPath, value); + timeValueMicros = ((uint64_t)value) * 24ull * 60ull * 60ull * 1000ull * 1000ull; + } else if (IsTimeUnitHours(suffix)) { + MOT_LOG_TRACE("Loaded %s: %u hours", cfgPath, value); + timeValueMicros = ((uint64_t)value) * 60ull * 60ull * 1000ull * 1000ull; + } else if (IsTimeUnitMinutes(suffix)) { + MOT_LOG_TRACE("Loaded %s: %u minutes", cfgPath, value); + timeValueMicros = ((uint64_t)value) * 60ull * 1000ull * 1000ull; + } else if (IsTimeUnitSeconds(suffix)) { + MOT_LOG_TRACE("Loaded %s: %u seconds", cfgPath, value); + timeValueMicros = ((uint64_t)value) * 1000ull * 1000ull; + } else if (IsTimeUnitMilliSeconds(suffix)) { + MOT_LOG_TRACE("Loaded %s: %u milli-seconds", cfgPath, value); + timeValueMicros = ((uint64_t)value) * 1000ull; + } else { + MOT_LOG_WARN("Invalid %s time value format: %s (invalid unit specifier '%s' - should be one of d, h, " + "m, s or ms)", + cfgPath, + timeValue, + suffix.c_str()); + } } } return timeValueMicros; diff --git a/src/gausskernel/storage/mot/core/system/transaction_logger/group_synchronous_redo_log/commit_group.cpp b/src/gausskernel/storage/mot/core/system/transaction_logger/group_synchronous_redo_log/commit_group.cpp index 6c05e6204..3d503d80c 100644 --- a/src/gausskernel/storage/mot/core/system/transaction_logger/group_synchronous_redo_log/commit_group.cpp +++ b/src/gausskernel/storage/mot/core/system/transaction_logger/group_synchronous_redo_log/commit_group.cpp @@ -26,7 +26,7 @@ #include "commit_group.h" #include "utilities.h" -#include "group_synchronous_redo_log_handler.h" +#include "group_sync_redo_log_handler.h" namespace MOT { DECLARE_LOGGER(CommitGroup, RedoLog); @@ -52,6 +52,15 @@ CommitGroup::CommitGroup(RedoLogBuffer* buffer, GroupSyncRedoLogHandler* handler CommitGroup::~CommitGroup() {} +/* function name:AddToGroup + function purpose:Add transaction to transaction commit group + input:Redo Log Buffer + output:Returns the position of the Redo Log Buffer in the Commit Group; + note:no + annotator:liushifa + annotate time:2023/09/11 17:38:20 + contact:3325287047@qq.com +*/ int CommitGroup::AddToGroup(RedoLogBuffer* buffer) { int val; @@ -62,7 +71,7 @@ int CommitGroup::AddToGroup(RedoLogBuffer* buffer) val = (m_groupSize.fetch_add(1)); if (val >= (int)m_maxGroupCommitSize) { // group is full... undo our add - m_groupSize.fetch_sub(1); + (void)m_groupSize.fetch_sub(1); m_rwlock.RdUnlock(); return -1; } @@ -76,10 +85,10 @@ int CommitGroup::AddToGroup(RedoLogBuffer* buffer) void CommitGroup::LogGroup() { ILogger* logger = m_handler->GetLogger(); - logger->AddToLog(m_groupData, m_groupSize); + (void)logger->AddToLog(m_groupData, m_groupSize); logger->FlushLog(); m_commited = true; - MOT_LOG_DEBUG("group committed. num entries: %d, handler id: %d", m_groupSize, m_handlerId); + MOT_LOG_DEBUG("group committed. num entries: %d, handler id: %d", m_groupSize.load(), m_handlerId); } void CommitGroup::Commit(bool isLeader, std::shared_ptr groupRef) @@ -94,9 +103,9 @@ void CommitGroup::Commit(bool isLeader, std::shared_ptr groupRef) void CommitGroup::WaitLeader(std::shared_ptr groupRef) { - m_numWaiters.fetch_add(1); + (void)m_numWaiters.fetch_add(1); std::unique_lock lock(m_fullGroupMutex); - m_fullGroupCV.wait_for(lock, m_groupTimeout, [this] { return m_groupSize >= m_maxGroupCommitSize; }); + (void)m_fullGroupCV.wait_for(lock, m_groupTimeout, [this] { return m_groupSize >= m_maxGroupCommitSize; }); m_rwlock.WrLock(); m_closed = true; m_handler->CloseGroup(groupRef); @@ -110,9 +119,9 @@ void CommitGroup::WaitMember() if (val == m_maxGroupCommitSize - 1) { // i was the last one, all group txns are waiting MOT_LOG_DEBUG("all in, releasing group to commit. group size: %d, " "handler id: %d", - m_groupSize, + m_groupSize.load(), m_handlerId); - std::lock_guard lock(m_fullGroupMutex); + std::lock_guard groupLock(m_fullGroupMutex); m_fullGroupCV.notify_all(); } m_groupCommitedCV.wait(lock, [this] { return m_commited; }); diff --git a/src/gausskernel/storage/mot/fdw_adapter/mot_fdw.cpp b/src/gausskernel/storage/mot/fdw_adapter/mot_fdw.cpp index 8113b36c3..bdabdcf2d 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/mot_fdw.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/mot_fdw.cpp @@ -416,44 +416,82 @@ static void MemoryEreportError() } } -/* - * - */ + +/* function name: MOTGetForeignRelSize + function purpose:Estimate and set the size and cost information of an external data table (Foreign Table). + This information is usually used in the query plan generation phase to determine the shape and execution strategy of the query plan. + input: + 'Planner Info* root': This is a pointer to the Planner Info structure, which is usually used to store query-related information during + the query optimization phase. It contains global information about the entire query, such as the target list of the query, the scope of + the query, and so on. + 'RelOptInfo* baserel':This is a pointer to a Rel Opt Info structure that describes which relations in a query can be optimized. The relation + here is a core concept in a relational database, which can be simply understood as a table. Rel Opt Info usually includes information about + the estimated size of the relationship, available access paths (Path), and so on. + 'Oid foreigntableid':This is an identifier for an external table, often referred to as an Object Identifier (OID for short). OID is a system + data type used to assign a unique identifier to each database object, such as table, index, etc. Here, foreigntableid is used to identify + the foreign table to be operated. + output:NULL + note:NULL + annotator:liushifa + annotate time:2023/08/02 15:20:24 + contact:3325287047@qq.com +*/ static void MOTGetForeignRelSize(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid) { + // Allocate memory for an instance of the MOTFdwStateSt structure MOTFdwStateSt* planstate = (MOTFdwStateSt*)palloc0(sizeof(MOTFdwStateSt)); + + // Get the foreign table data using the foreign table id ForeignTable* ftable = GetForeignTable(foreigntableid); + + // Get the current transaction manager MOT::TxnManager* currTxn = GetSafeTxn(__FUNCTION__); + + // Declare some variables for later use Bitmapset* attrs = nullptr; ListCell* lc = nullptr; bool needWholeRow = false; TupleDesc desc; + + // Get the Relation object for the foreign table Relation rel = RelationIdGetRelation(ftable->relid); + // Get the table from the current transaction using the Relation object's id planstate->m_table = currTxn->GetTableByExternalId(RelationGetRelid(rel)); + + // If the table is not found in the MOT engine, abort the transaction if (planstate->m_table == nullptr) { abortParentTransactionParamsNoDetail( ERRCODE_UNDEFINED_TABLE, MOT_TABLE_NOTFOUND, (char*)RelationGetRelationName(rel)); return; } + + // Save the planstate to the base relation object's private field baserel->fdw_private = planstate; + + // Initialize some fields in the planstate planstate->m_hasForUpdate = root->parse->hasForUpdate; planstate->m_cmdOper = root->parse->commandType; planstate->m_foreignTableId = foreigntableid; + desc = RelationGetDescr(rel); planstate->m_numAttrs = RelationGetNumberOfAttributes(rel); + + // More initialization of the planstate int len = BITMAP_GETLEN(planstate->m_numAttrs); planstate->m_attrsUsed = (uint8_t*)palloc0(len); planstate->m_attrsModified = (uint8_t*)palloc0(len); + + // Check whether a whole row is needed based on whether there are any AFTER ROW INSERT triggers needWholeRow = rel->trigdesc && rel->trigdesc->trig_insert_after_row; + // If a whole row is not needed, pull the needed columns from the base relation + // Otherwise, set all columns as used foreach (lc, baserel->baserestrictinfo) { RestrictInfo* ri = (RestrictInfo*)lfirst(lc); - if (!needWholeRow) pull_varattnos((Node*)ri->clause, baserel->relid, &attrs); } - if (needWholeRow) { for (int i = 0; i < desc->natts; i++) { if (!desc->attrs[i]->attisdropped) { @@ -472,17 +510,23 @@ static void MOTGetForeignRelSize(PlannerInfo* root, RelOptInfo* baserel, Oid for } } } - + // Get the row count of the table from the MOT engine and set it to the base relation baserel->rows = planstate->m_table->GetRowCount(); baserel->tuples = planstate->m_table->GetRowCount(); + + // If the row count is zero, default it to 100,000 if (baserel->rows == 0) baserel->rows = baserel->tuples = 100000; + + // Set the startup cost and total cost of the plan state planstate->m_startupCost = 0.1; planstate->m_totalCost = baserel->rows * planstate->m_startupCost; + // Close the Relation object RelationClose(rel); } + static bool IsOrderingApplicable(PathKey* pathKey, RelOptInfo* rel, MOT::Index* ix, OrderSt* ord) { bool res = false; @@ -531,11 +575,27 @@ static bool IsOrderingApplicable(PathKey* pathKey, RelOptInfo* rel, MOT::Index* return res; } -/* - * - */ +/* function name:MOTGetForeignPaths + function purpose:Called in case of an index to determine which indexes are available for fetching data from + the table in the current query + input:'Planner Info* root': This is a pointer to the Planner Info structure, which is usually used to store query-related information during + the query optimization phase. It contains global information about the entire query, such as the target list of the query, the scope of + the query, and so on. + 'RelOptInfo* baserel':This is a pointer to a Rel Opt Info structure that describes which relations in a query can be optimized. The relation + here is a core concept in a relational database, which can be simply understood as a table. Rel Opt Info usually includes information about + the estimated size of the relationship, available access paths (Path), and so on. + 'Oid foreigntableid':This is an identifier for an external table, often referred to as an Object Identifier (OID for short). OID is a system + data type used to assign a unique identifier to each database object, such as table, index, etc. Here, foreigntableid is used to identify + the foreign table to be operated. + output:NULL + note:NULL + annotator:liushifa + annotate time:2023/08/02 15:29:51 + contact:3325287047@qq.com +*/ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid) { + // Retrieve the MOTFdwStateSt structure from the base relation object MOTFdwStateSt* planstate = (MOTFdwStateSt*)baserel->fdw_private; List* usablePathkeys = NIL; List* bestClause = nullptr; @@ -546,17 +606,15 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei Path* fpReg = nullptr; Path* fpIx = nullptr; bool hasRegularPath = false; - planstate->m_order = SORTDIR_ENUM::SORTDIR_ASC; // first create regular path based on relation restrictions foreach (lc, baserel->baserestrictinfo) { RestrictInfo* ri = (RestrictInfo*)lfirst(lc); - + // If the expression is not a MOT expression, add it to local conditions if (!IsMOTExpr(baserel, planstate, &marr, ri->clause, nullptr, true)) { planstate->m_localConds = lappend(planstate->m_localConds, ri->clause); } } - // get best index best = MOTAdaptor::GetBestMatchIndex(planstate, &marr, list_length(baserel->baserestrictinfo)); if (best != nullptr) { @@ -569,7 +627,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei planstate->m_startupCost = 0.001; planstate->m_totalCost = best->m_cost; planstate->m_bestIx = best; - foreach (lc, root->query_pathkeys) { PathKey* pathkey = (PathKey*)lfirst(lc); @@ -577,7 +634,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei usablePathkeys = lappend(usablePathkeys, pathkey); } } - if (!best->CanApplyOrdering(ord.m_cols)) { list_free(usablePathkeys); usablePathkeys = nullptr; @@ -591,12 +647,10 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei ord.init(); MOT::Index* ix = planstate->m_table->GetPrimaryIndex(); List* keys; - if (root->query_level == 1) keys = root->query_pathkeys; else keys = root->sort_pathkeys; - foreach (lc, keys) { PathKey* pathkey = (PathKey*)lfirst(lc); @@ -604,7 +658,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei usablePathkeys = lappend(usablePathkeys, pathkey); } } - if (list_length(usablePathkeys) > 0) { if (ord.m_cols[0] != 0) planstate->m_order = ord.m_order; @@ -615,7 +668,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei } else planstate->m_order = SORTDIR_ENUM::SORTDIR_ASC; } - fpReg = (Path*)create_foreignscan_path(root, baserel, planstate->m_startupCost, @@ -624,7 +676,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei nullptr, /* no outer rel either */ nullptr, // private data will be assigned later 0); - foreach (lc, baserel->pathlist) { Path* path = (Path*)lfirst(lc); if (IsA(path, IndexPath) && path->param_info == nullptr) { @@ -635,7 +686,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei if (!hasRegularPath) add_path(root, baserel, fpReg); set_cheapest(baserel); - if (!IS_PGXC_COORDINATOR && list_length(baserel->cheapest_parameterized_paths) > 0) { foreach (lc, baserel->cheapest_parameterized_paths) { bestPath = (Path*)lfirst(lc); @@ -647,7 +697,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei } usablePathkeys = nullptr; } - if (bestClause != nullptr) { marr.Clear(); @@ -657,7 +706,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei // In case we use index params DO NOT add it to envelope filter. (void)IsMOTExpr(baserel, planstate, &marr, ri->clause, nullptr, false); } - best = MOTAdaptor::GetBestMatchIndex(planstate, &marr, list_length(bestClause), false); if (best != nullptr) { OrderSt ord; @@ -697,7 +745,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei fpIx->param_info = bestPath->param_info; } } - List* newPath = nullptr; List* origPath = baserel->pathlist; // disable index path @@ -708,7 +755,6 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei else pfree(path); } - list_free(origPath); baserel->pathlist = newPath; if (hasRegularPath) @@ -717,10 +763,23 @@ static void MOTGetForeignPaths(PlannerInfo* root, RelOptInfo* baserel, Oid forei add_path(root, baserel, fpIx); set_cheapest(baserel); } - -/* - * - */ +/* function name:MOTGetForeignPlan + function purpose:Create an execution plan for fetching data from the table + input:'Planner Info* root': This is a pointer to the Planner Info structure, which is usually used to store query-related information during + the query optimization phase. It contains global information about the entire query, such as the target list of the query, the scope of + the query, and so on. + 'RelOptInfo* baserel':This is a pointer to a Rel Opt Info structure that describes which relations in a query can be optimized. The relation + here is a core concept in a relational database, which can be simply understood as a table. Rel Opt Info usually includes information about + the estimated size of the relationship, available access paths (Path), and so on. + 'Oid foreigntableid':This is an identifier for an external table, often referred to as an Object Identifier (OID for short). OID is a system + data type used to assign a unique identifier to each database object, such as table, index, etc. Here, foreigntableid is used to identify + the foreign table to be operated. + output:ForeignScan plan for a foreign table. + note: + annotator:liushifa + annotate time:2023/08/02 15:44:43 + contact:3325287047@qq.com +*/ static ForeignScan* MOTGetForeignPlan( PlannerInfo* root, RelOptInfo* baserel, Oid foreigntableid, ForeignPath* best_path, List* tlist, List* scan_clauses) { @@ -729,7 +788,8 @@ static ForeignScan* MOTGetForeignPlan( MOTFdwStateSt* planstate = (MOTFdwStateSt*)baserel->fdw_private; List* tmpLocal = nullptr; List* remote = nullptr; - + // If there is a parameterized path and a best index match, clean and free the old best index match + // Then set the best index match to the parameterized one if (best_path->path.param_info && planstate->m_paramBestIx) { if (planstate->m_bestIx != nullptr) { planstate->m_bestIx->Clean(planstate); @@ -738,7 +798,8 @@ static ForeignScan* MOTGetForeignPlan( planstate->m_bestIx = planstate->m_paramBestIx; planstate->m_paramBestIx = nullptr; } - + // If there is a best index match, concatenate the remote conditions and set the number of expressions + // If there is no best index match, set the number of expressions to 0 if (planstate->m_bestIx != nullptr) { planstate->m_numExpr = list_length(planstate->m_bestIx->m_remoteConds); remote = list_concat(planstate->m_bestIx->m_remoteConds, planstate->m_bestIx->m_remoteCondsOrig); @@ -750,6 +811,7 @@ static ForeignScan* MOTGetForeignPlan( } else { planstate->m_numExpr = 0; } + // Initialize local conditions baserel->fdw_private = nullptr; tmpLocal = planstate->m_localConds; planstate->m_localConds = nullptr; @@ -785,6 +847,8 @@ static ForeignScan* MOTGetForeignPlan( list_free(tmpLocal); List* quals = planstate->m_localConds; + // Create a ForeignScan node with the target list, local conditions, scan relation ID, and other information + // make_foreignscan is defined in /openGauss-server/src/gausskernel/optimizer/plan/createplan.cpp return make_foreignscan(tlist, quals, scanRelid, diff --git a/src/gausskernel/storage/mot/fdw_adapter/mot_internal.cpp b/src/gausskernel/storage/mot/fdw_adapter/mot_internal.cpp index 7483d01a8..a52c38347 100644 --- a/src/gausskernel/storage/mot/fdw_adapter/mot_internal.cpp +++ b/src/gausskernel/storage/mot/fdw_adapter/mot_internal.cpp @@ -29,7 +29,6 @@ #include #include "postgres.h" -#include "access/dfs/dfs_query.h" #include "access/sysattr.h" #include "nodes/parsenodes.h" #include "nodes/pg_list.h" @@ -41,8 +40,10 @@ #include "storage/ipc.h" #include "commands/dbcommands.h" #include "knl/knl_session.h" +#include "utils/date.h" #include "mot_internal.h" +#include "mot_fdw_helpers.h" #include "row.h" #include "log_statistics.h" #include "spin_lock.h" @@ -65,30 +66,25 @@ #include "jit_statistics.h" #include "gaussdb_config_loader.h" -#define IS_CHAR_TYPE(oid) (oid == VARCHAROID || oid == BPCHAROID || oid == TEXTOID || oid == CLOBOID || oid == BYTEAOID) -#define IS_INT_TYPE(oid) \ - (oid == BOOLOID || oid == CHAROID || oid == INT8OID || oid == INT2OID || oid == INT4OID || oid == FLOAT4OID || \ - oid == FLOAT8OID || oid == INT1OID || oid == DATEOID || oid == TIMEOID || oid == TIMESTAMPOID || \ - oid == TIMESTAMPTZOID) - MOT::MOTEngine* MOTAdaptor::m_engine = nullptr; -static XLOGLogger xlogger; +static XLOGLogger g_xlogger; +static SnapshotManager g_snapshotMgr; // enable MOT Engine logging facilities DECLARE_LOGGER(InternalExecutor, FDW) -/** @brief on_proc_exit() callback for cleaning up current thread - only when thread pool is ENABLED. */ -static void MOTCleanupThread(int status, Datum ptr); - -/** @brief Helper for cleaning up all JIT context objects stored in all CachedPlanSource of the current session. */ -static void DestroySessionJitContexts(); - // in a thread-pooled environment we need to ensure thread-locals are initialized properly -static inline void EnsureSafeThreadAccessInline() +static inline bool EnsureSafeThreadAccessInline(bool throwError = true) { if (MOTCurrThreadId == INVALID_THREAD_ID) { MOT_LOG_DEBUG("Initializing safe thread access for current thread"); - MOT::AllocThreadId(); + if (MOT::AllocThreadId() == INVALID_THREAD_ID) { + MOT_LOG_ERROR("Failed to allocate thread identifier"); + if (throwError) { + ereport(ERROR, (errmodule(MOD_MOT), errmsg("Failed to allocate thread identifier"))); + } + return false; + } // register for cleanup only once - not having a current thread id is the safe indicator we never registered // proc-exit callback for this thread if (g_instance.attr.attr_common.enable_thread_pool) { @@ -97,134 +93,27 @@ static inline void EnsureSafeThreadAccessInline() } } if (MOTCurrentNumaNodeId == MEM_INVALID_NODE) { - MOT::InitCurrentNumaNodeId(); - } - MOT::InitMasstreeThreadinfo(); -} - -extern void EnsureSafeThreadAccess() -{ - EnsureSafeThreadAccessInline(); -} - -static void DestroySession(MOT::SessionContext* sessionContext) -{ - MOT_ASSERT(MOTAdaptor::m_engine); - MOT_LOG_DEBUG("Destroying session context %p, connection_id %u", sessionContext, sessionContext->GetConnectionId()); - - if (u_sess->mot_cxt.jit_session_context_pool) { - JitExec::FreeSessionJitContextPool(u_sess->mot_cxt.jit_session_context_pool); - } - MOT::GetSessionManager()->DestroySessionContext(sessionContext); -} - -// Global map of PG session identification (required for session statistics) -// This approach is safer than saving information in the session context -static pthread_spinlock_t sessionDetailsLock; -typedef std::map> SessionDetailsMap; -static SessionDetailsMap sessionDetailsMap; - -static void InitSessionDetailsMap() -{ - pthread_spin_init(&sessionDetailsLock, 0); -} - -static void DestroySessionDetailsMap() -{ - pthread_spin_destroy(&sessionDetailsLock); -} - -static void RecordSessionDetails() -{ - MOT::SessionId sessionId = u_sess->mot_cxt.session_id; - if (sessionId != INVALID_SESSION_ID) { - pthread_spin_lock(&sessionDetailsLock); - sessionDetailsMap.emplace(sessionId, std::make_pair(t_thrd.proc->pid, t_thrd.proc->myStartTime)); - pthread_spin_unlock(&sessionDetailsLock); - } -} - -static void ClearSessionDetails(MOT::SessionId sessionId) -{ - if (sessionId != INVALID_SESSION_ID) { - pthread_spin_lock(&sessionDetailsLock); - SessionDetailsMap::iterator itr = sessionDetailsMap.find(sessionId); - if (itr != sessionDetailsMap.end()) { - sessionDetailsMap.erase(itr); - } - pthread_spin_unlock(&sessionDetailsLock); - } -} - -inline void ClearCurrentSessionDetails() -{ - ClearSessionDetails(u_sess->mot_cxt.session_id); -} - -static void GetSessionDetails(MOT::SessionId sessionId, ::ThreadId* gaussSessionId, pg_time_t* sessionStartTime) -{ - // although we have the PGPROC in the user data of the session context, we prefer not to use - // it due to safety (in some unknown constellation we might hold an invalid pointer) - // it is much safer to save a copy of the two required fields - pthread_spin_lock(&sessionDetailsLock); - SessionDetailsMap::iterator itr = sessionDetailsMap.find(sessionId); - if (itr != sessionDetailsMap.end()) { - *gaussSessionId = itr->second.first; - *sessionStartTime = itr->second.second; - } - pthread_spin_unlock(&sessionDetailsLock); -} - -// provide safe session auto-cleanup in case of missing session closure -// This mechanism relies on the fact that when a session ends, eventually its thread is terminated -// ATTENTION: in thread-pooled envelopes this assumption no longer holds true, since the container thread keeps -// running after the session ends, and a session might run each time on a different thread, so we -// disable this feature, instead we use this mechanism to generate thread-ended event into the MOT Engine -static pthread_key_t sessionCleanupKey; - -static void SessionCleanup(void* key) -{ - MOT_ASSERT(!g_instance.attr.attr_common.enable_thread_pool); - - // in order to ensure session-id cleanup for session 0 we use positive values - MOT::SessionId sessionId = (MOT::SessionId)(((uint64_t)key) - 1); - if (sessionId != INVALID_SESSION_ID) { - MOT_LOG_WARN("Encountered unclosed session %u (missing call to DestroyTxn()?)", (unsigned)sessionId); - ClearSessionDetails(sessionId); - MOT_LOG_DEBUG("SessionCleanup(): Calling DestroySessionJitContext()"); - DestroySessionJitContexts(); - if (MOTAdaptor::m_engine) { - MOT::SessionContext* sessionContext = MOT::GetSessionManager()->GetSessionContext(sessionId); - if (sessionContext != nullptr) { - DestroySession(sessionContext); + if (!MOT::InitCurrentNumaNodeId()) { + MOT_LOG_ERROR("Failed to allocate NUMA node identifier"); + if (throwError) { + ereport(ERROR, (errmodule(MOD_MOT), errmsg("Failed to allocate NUMA node identifier"))); } - // since a call to on_proc_exit(destroyTxn) was probably missing, we should also cleanup thread-locals - // pay attention that if we got here it means the thread pool is disabled, so we must ensure thread-locals - // are cleaned up right now. Due to these complexities, onCurrentThreadEnding() was designed to be proof - // for repeated calls. - MOTAdaptor::m_engine->OnCurrentThreadEnding(); + return false; } } + if (!MOT::InitMasstreeThreadinfo()) { + MOT_LOG_ERROR("Failed to initialize thread-local masstree info"); + if (throwError) { + ereport(ERROR, (errmodule(MOD_MOT), errmsg("Failed to initialize thread-local masstree info"))); + } + return false; + } + return true; } -static void InitSessionCleanup() +extern bool EnsureSafeThreadAccess(bool throwError /* = true */) { - pthread_key_create(&sessionCleanupKey, SessionCleanup); -} - -static void DestroySessionCleanup() -{ - pthread_key_delete(sessionCleanupKey); -} - -static void ScheduleSessionCleanup() -{ - pthread_setspecific(sessionCleanupKey, (const void*)(uint64_t)(u_sess->mot_cxt.session_id + 1)); -} - -static void CancelSessionCleanup() -{ - pthread_setspecific(sessionCleanupKey, nullptr); + return EnsureSafeThreadAccessInline(throwError); } static GaussdbConfigLoader* gaussdbConfigLoader = nullptr; @@ -268,7 +157,7 @@ void MOTAdaptor::Init() } if (!m_engine->LoadConfig()) { - m_engine->RemoveConfigLoader(gaussdbConfigLoader); + (void)m_engine->RemoveConfigLoader(gaussdbConfigLoader); delete gaussdbConfigLoader; gaussdbConfigLoader = nullptr; MOT::MOTEngine::DestroyInstance(); @@ -285,17 +174,17 @@ void MOTAdaptor::Init() if ((g_instance.attr.attr_memory.max_process_memory < (int32)maxReserveMemoryKb) || ((g_instance.attr.attr_memory.max_process_memory - maxReserveMemoryKb) < MIN_DYNAMIC_PROCESS_MEMORY)) { // we allow one extreme case: GaussDB is configured to its limit, and zero memory is left for us - if (maxReserveMemoryKb <= motCfg.MOT_MIN_MEMORY_USAGE_MB * KILO_BYTE) { + if (maxReserveMemoryKb <= MOT::MOTConfiguration::MOT_MIN_MEMORY_USAGE_MB * KILO_BYTE) { MOT_LOG_WARN("Allowing MOT to work in minimal memory mode"); } else { - m_engine->RemoveConfigLoader(gaussdbConfigLoader); + (void)m_engine->RemoveConfigLoader(gaussdbConfigLoader); delete gaussdbConfigLoader; gaussdbConfigLoader = nullptr; MOT::MOTEngine::DestroyInstance(); elog(FATAL, "The value of pre-reserved memory for MOT engine is not reasonable: " "Request for a maximum of %" PRIu64 " KB global memory, and %" PRIu64 - " KB session memory (total of %" PRIu64 " KB) is invalid since max_process_memory is %u KB", + " KB session memory (total of %" PRIu64 " KB) is invalid since max_process_memory is %d KB", globalMemoryKb, localMemoryKb, maxReserveMemoryKb, @@ -304,7 +193,7 @@ void MOTAdaptor::Init() } if (!m_engine->Initialize()) { - m_engine->RemoveConfigLoader(gaussdbConfigLoader); + (void)m_engine->RemoveConfigLoader(gaussdbConfigLoader); delete gaussdbConfigLoader; gaussdbConfigLoader = nullptr; MOT::MOTEngine::DestroyInstance(); @@ -312,7 +201,7 @@ void MOTAdaptor::Init() } if (!JitExec::JitStatisticsProvider::CreateInstance()) { - m_engine->RemoveConfigLoader(gaussdbConfigLoader); + (void)m_engine->RemoveConfigLoader(gaussdbConfigLoader); delete gaussdbConfigLoader; gaussdbConfigLoader = nullptr; MOT::MOTEngine::DestroyInstance(); @@ -320,10 +209,17 @@ void MOTAdaptor::Init() } // make sure current thread is cleaned up properly when thread pool is enabled - EnsureSafeThreadAccessInline(); + // avoid throwing errors on failure + if (!EnsureSafeThreadAccessInline(false)) { + (void)m_engine->RemoveConfigLoader(gaussdbConfigLoader); + delete gaussdbConfigLoader; + gaussdbConfigLoader = nullptr; + MOT::MOTEngine::DestroyInstance(); + elog(FATAL, "Failed to initialize thread-local data."); + } if (motCfg.m_enableRedoLog && motCfg.m_loggerType == MOT::LoggerType::EXTERNAL_LOGGER) { - m_engine->GetRedoLogHandler()->SetLogger(&xlogger); + m_engine->GetRedoLogHandler()->SetLogger(&g_xlogger); m_engine->GetRedoLogHandler()->SetWalWakeupFunc(WakeupWalWriter); } @@ -332,6 +228,10 @@ void MOTAdaptor::Init() InitSessionCleanup(); } InitKeyOperStateMachine(); + + MOT_LOG_INFO("Switching to External snapshot manager"); + m_engine->SetCSNManager(&g_snapshotMgr); + m_initialized = true; } @@ -354,12 +254,13 @@ void MOTAdaptor::Destroy() } DestroySessionDetailsMap(); if (gaussdbConfigLoader != nullptr) { - m_engine->RemoveConfigLoader(gaussdbConfigLoader); + (void)m_engine->RemoveConfigLoader(gaussdbConfigLoader); delete gaussdbConfigLoader; gaussdbConfigLoader = nullptr; } - EnsureSafeThreadAccessInline(); + // avoid throwing errors and ignore them at this phase + (void)EnsureSafeThreadAccessInline(false); MOT::MOTEngine::DestroyInstance(); m_engine = nullptr; knl_thread_mot_init(); // reset all thread-locals, mandatory for standby switch-over @@ -410,89 +311,12 @@ MOT::TxnManager* MOTAdaptor::InitTxnManager( } u_sess->mot_cxt.txn_manager = session_ctx->GetTxnManager(); - elog(DEBUG1, "Init TXN_MAN for thread %u", MOTCurrThreadId); + elog(DEBUG1, "Init TXN_MAN for thread %" PRIu16, MOTCurrThreadId); } return u_sess->mot_cxt.txn_manager; } -static void DestroySessionJitContexts() -{ - // we must release all JIT context objects associated with this session now. - // it seems that when thread pool is disabled, all cached plan sources for the session are not - // released explicitly, but rather implicitly as part of the release of the memory context of the session. - // in any case, we guard against repeated destruction of the JIT context by nullifying it - MOT_LOG_DEBUG("Cleaning up all JIT context objects for current session"); - CachedPlanSource* psrc = u_sess->pcache_cxt.first_saved_plan; - while (psrc != nullptr) { - if (psrc->mot_jit_context != nullptr) { - MOT_LOG_DEBUG("DestroySessionJitContexts(): Calling DestroyJitContext(%p)", psrc->mot_jit_context); - JitExec::DestroyJitContext(psrc->mot_jit_context); - psrc->mot_jit_context = nullptr; - } - psrc = psrc->next_saved; - } - MOT_LOG_DEBUG("DONE Cleaning up all JIT context objects for current session"); -} - -/** @brief Notification from thread pool that a session ended (only when thread pool is ENABLED). */ -extern void MOTOnSessionClose() -{ - MOT_LOG_TRACE("Received session close notification (current session id: %u, current connection id: %u)", - u_sess->mot_cxt.session_id, - u_sess->mot_cxt.connection_id); - if (u_sess->mot_cxt.session_id != INVALID_SESSION_ID) { - ClearCurrentSessionDetails(); - MOT_LOG_DEBUG("MOTOnSessionClose(): Calling DestroySessionJitContexts()"); - DestroySessionJitContexts(); - if (!MOTAdaptor::m_engine) { - MOT_LOG_ERROR("MOTOnSessionClose(): MOT engine is not initialized"); - } else { - EnsureSafeThreadAccessInline(); // this is ok, it wil be cleaned up when thread exits - MOT::SessionContext* sessionContext = u_sess->mot_cxt.session_context; - if (sessionContext == nullptr) { - MOT_LOG_WARN("Received session close notification, but no current session is found. Current session id " - "is %u. Request ignored.", - u_sess->mot_cxt.session_id); - } else { - DestroySession(sessionContext); - MOT_ASSERT(u_sess->mot_cxt.session_id == INVALID_SESSION_ID); - } - } - } -} - -/** @brief Notification from thread pool that a pooled thread ended (only when thread pool is ENABLED). */ -static void MOTOnThreadShutdown() -{ - if (!MOTAdaptor::m_initialized) { - return; - } - - MOT_LOG_TRACE("Received thread shutdown notification"); - if (!MOTAdaptor::m_engine) { - MOT_LOG_ERROR("MOTOnThreadShutdown(): MOT engine is not initialized"); - } else { - MOTAdaptor::m_engine->OnCurrentThreadEnding(); - } - knl_thread_mot_init(); // reset all thread locals -} - -/** - * @brief on_proc_exit() callback to handle thread-cleanup - regardless of whether thread pool is enabled or not. - * registration to on_proc_exit() is triggered by first call to EnsureSafeThreadAccessInline(). - */ -static void MOTCleanupThread(int status, Datum ptr) -{ - MOT_ASSERT(g_instance.attr.attr_common.enable_thread_pool); - MOT_LOG_TRACE("Received thread cleanup notification (thread-pool ON)"); - - // when thread pool is used we just cleanup current thread - // this might be a duplicate because thread pool also calls MOTOnThreadShutdown() - this is still ok - // because we guard against repeated calls in MOTEngine::onCurrentThreadEnding() - MOTOnThreadShutdown(); -} - void MOTAdaptor::DestroyTxn(int status, Datum ptr) { MOT_ASSERT(!g_instance.attr.attr_common.enable_thread_pool); @@ -512,8 +336,10 @@ void MOTAdaptor::DestroyTxn(int status, Datum ptr) if (session != MOT_GET_CURRENT_SESSION_CONTEXT()) { MOT_LOG_WARN("Ignoring request to delete session context: already deleted"); } else if (session != nullptr) { - elog(DEBUG1, "Destroy SessionContext, connection_id = %u \n", session->GetConnectionId()); - EnsureSafeThreadAccessInline(); // may be accessed from new thread pool worker + elog(DEBUG1, "Destroy SessionContext, connection_id = %u", session->GetConnectionId()); + // initialize thread data, since this call may be accessed from new thread pool worker + // avoid throwing errors and ignore them at this phase + (void)EnsureSafeThreadAccessInline(false); MOT::GcManager* gc = MOT_GET_CURRENT_SESSION_CONTEXT()->GetTxnManager()->GetGcSession(); if (gc != nullptr) { gc->GcEndTxn(); @@ -527,7 +353,7 @@ void MOTAdaptor::DestroyTxn(int status, Datum ptr) MOT::RC MOTAdaptor::ValidateCommit() { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); if (!IS_PGXC_COORDINATOR) { return txn->ValidateCommit(); @@ -539,7 +365,7 @@ MOT::RC MOTAdaptor::ValidateCommit() void MOTAdaptor::RecordCommit(uint64_t csn) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); txn->SetCommitSequenceNumber(csn); if (!IS_PGXC_COORDINATOR) { @@ -551,7 +377,7 @@ void MOTAdaptor::RecordCommit(uint64_t csn) MOT::RC MOTAdaptor::Commit(uint64_t csn) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); txn->SetCommitSequenceNumber(csn); if (!IS_PGXC_COORDINATOR) { @@ -562,9 +388,19 @@ MOT::RC MOTAdaptor::Commit(uint64_t csn) } } +bool MOTAdaptor::IsTxnWriteSetEmpty() +{ + (void)EnsureSafeThreadAccessInline(); + MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); + if (txn->m_txnDdlAccess->Size() > 0 or txn->m_accessMgr->Size() > 0) { + return false; + } + return true; +} + void MOTAdaptor::EndTransaction() { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); // Nothing to do in coordinator if (!IS_PGXC_COORDINATOR) { @@ -574,7 +410,7 @@ void MOTAdaptor::EndTransaction() void MOTAdaptor::Rollback() { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); if (!IS_PGXC_COORDINATOR) { txn->Rollback(); @@ -585,7 +421,7 @@ void MOTAdaptor::Rollback() MOT::RC MOTAdaptor::Prepare() { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); if (!IS_PGXC_COORDINATOR) { return txn->Prepare(); @@ -597,7 +433,7 @@ MOT::RC MOTAdaptor::Prepare() void MOTAdaptor::CommitPrepared(uint64_t csn) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); txn->SetCommitSequenceNumber(csn); if (!IS_PGXC_COORDINATOR) { @@ -609,7 +445,7 @@ void MOTAdaptor::CommitPrepared(uint64_t csn) void MOTAdaptor::RollbackPrepared() { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); if (!IS_PGXC_COORDINATOR) { txn->RollbackPrepared(); @@ -620,7 +456,7 @@ void MOTAdaptor::RollbackPrepared() MOT::RC MOTAdaptor::InsertRow(MOTFdwStateSt* fdwState, TupleTableSlot* slot) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); uint8_t* newRowData = nullptr; fdwState->m_currTxn->SetTransactionId(fdwState->m_txnId); MOT::Table* table = fdwState->m_table; @@ -643,8 +479,10 @@ MOT::RC MOTAdaptor::InsertRow(MOTFdwStateSt* fdwState, TupleTableSlot* slot) MOT::RC MOTAdaptor::UpdateRow(MOTFdwStateSt* fdwState, TupleTableSlot* slot, MOT::Row* currRow) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::RC rc; + MOT::TxnIxColUpdate colUpd( + fdwState->m_table, fdwState->m_currTxn, fdwState->m_attrsModified, fdwState->m_hasIndexedColUpdate); do { fdwState->m_currTxn->SetTransactionId(fdwState->m_txnId); @@ -652,11 +490,25 @@ MOT::RC MOTAdaptor::UpdateRow(MOTFdwStateSt* fdwState, TupleTableSlot* slot, MOT if (rc != MOT::RC::RC_OK) { break; } - uint8_t* rowData = const_cast(currRow->GetData()); + MOT::Row* currDraft = fdwState->m_currTxn->GetLastAccessedDraft(); + if (unlikely(fdwState->m_hasIndexedColUpdate != MOT::UpdateIndexColumnType::UPDATE_COLUMN_NONE)) { + rc = colUpd.InitAndBuildOldKeys(currDraft); + if (rc != MOT::RC::RC_OK) { + break; + } + } + uint8_t* rowData = const_cast(currDraft->GetData()); PackUpdateRow(slot, fdwState->m_table, fdwState->m_attrsModified, rowData); MOT::BitmapSet modified_columns(fdwState->m_attrsModified, fdwState->m_table->GetFieldCount() - 1); - - rc = fdwState->m_currTxn->OverwriteRow(currRow, modified_columns); + if (unlikely(fdwState->m_hasIndexedColUpdate)) { + rc = colUpd.FilterColumnUpdate(currDraft); + if (rc != MOT::RC::RC_OK) { + break; + } + rc = fdwState->m_currTxn->UpdateRow(modified_columns, &colUpd); + } else { + rc = fdwState->m_currTxn->OverwriteRow(currDraft, modified_columns); + } } while (0); return rc; @@ -664,12 +516,21 @@ MOT::RC MOTAdaptor::UpdateRow(MOTFdwStateSt* fdwState, TupleTableSlot* slot, MOT MOT::RC MOTAdaptor::DeleteRow(MOTFdwStateSt* fdwState, TupleTableSlot* slot) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); fdwState->m_currTxn->SetTransactionId(fdwState->m_txnId); MOT::RC rc = fdwState->m_currTxn->DeleteLastRow(); return rc; } +bool MOTAdaptor::IsColumnIndexed(int16_t colId, MOT::Table* table) +{ + MOT::Column* col = table->GetField(colId); + if (col != nullptr && col->IsUsedByIndex()) { + return true; + } + return false; +} + // NOTE: colId starts from 1 bool MOTAdaptor::SetMatchingExpr( MOTFdwStateSt* state, MatchIndexArr* marr, int16_t colId, KEY_OPER op, Expr* expr, Expr* parent, bool set_local) @@ -705,8 +566,9 @@ MatchIndex* MOTAdaptor::GetBestMatchIndex(MOTFdwStateSt* festate, MatchIndexArr* double cost = marr->m_idx[i]->GetCost(numClauses); if (cost < bestCost) { if (bestI < MAX_NUM_INDEXES) { - if (marr->m_idx[i]->GetNumMatchedCols() < marr->m_idx[bestI]->GetNumMatchedCols()) + if (marr->m_idx[i]->GetNumMatchedCols() < marr->m_idx[bestI]->GetNumMatchedCols()) { continue; + } } bestCost = cost; bestI = i; @@ -727,13 +589,15 @@ MatchIndex* MOTAdaptor::GetBestMatchIndex(MOTFdwStateSt* festate, MatchIndexArr* if (j > 0 && best->m_opers[k][j - 1] != KEY_OPER::READ_KEY_EXACT && !list_member(festate->m_localConds, best->m_parentColMatch[k][j])) { - if (setLocal) + if (setLocal) { festate->m_localConds = lappend(festate->m_localConds, best->m_parentColMatch[k][j]); + } } } else if (!list_member(festate->m_localConds, best->m_parentColMatch[k][j]) && !list_member(best->m_remoteCondsOrig, best->m_parentColMatch[k][j])) { - if (setLocal) + if (setLocal) { festate->m_localConds = lappend(festate->m_localConds, best->m_parentColMatch[k][j]); + } best->m_colMatch[k][j] = nullptr; best->m_parentColMatch[k][j] = nullptr; } @@ -781,7 +645,7 @@ void MOTAdaptor::OpenCursor(Relation rel, MOTFdwStateSt* festate) bool forwardDirection = true; bool found = false; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); // GetTableByExternalId cannot return nullptr at this stage, because it is protected by envelope's table lock. festate->m_table = festate->m_currTxn->GetTableByExternalId(rel->rd_id); @@ -790,57 +654,39 @@ void MOTAdaptor::OpenCursor(Relation rel, MOTFdwStateSt* festate) // this scan all keys case // we need to open both cursors on start and end to prevent // infinite scan in case "insert into table A ... as select * from table A ... - if (festate->m_bestIx == nullptr) { - int fIx, bIx; - uint8_t* buf = nullptr; + if (festate->m_bestIx == nullptr || festate->m_bestIx->m_fullScan) { // assumption that primary index cannot be changed, can take it from // table and not look on ddl_access - MOT::Index* ix = festate->m_table->GetPrimaryIndex(); + MOT::Index* ix = festate->m_bestIx ? festate->m_bestIx->m_ix : festate->m_table->GetPrimaryIndex(); uint16_t keyLength = ix->GetKeyLength(); - if (festate->m_order == SORTDIR_ENUM::SORTDIR_ASC) { - fIx = 0; - bIx = 1; + if (festate->m_order == SortDir::SORTDIR_ASC) { + festate->m_cursor[0] = ix->Begin(festate->m_currTxn->GetThdId()); festate->m_forwardDirectionScan = true; + festate->m_cursor[1] = nullptr; } else { - fIx = 1; - bIx = 0; + festate->m_stateKey[0].InitKey(keyLength); + uint8_t* buf = festate->m_stateKey[0].GetKeyBuf(); + errno_t erc = memset_s(buf, keyLength, 0xff, keyLength); + securec_check(erc, "\0", "\0"); + festate->m_cursor[0] = + ix->Search(&festate->m_stateKey[0], false, false, festate->m_currTxn->GetThdId(), found); festate->m_forwardDirectionScan = false; + festate->m_cursor[1] = nullptr; } - - festate->m_cursor[fIx] = festate->m_table->Begin(festate->m_currTxn->GetThdId()); - - festate->m_stateKey[bIx].InitKey(keyLength); - buf = festate->m_stateKey[bIx].GetKeyBuf(); - errno_t erc = memset_s(buf, keyLength, 0xff, keyLength); - securec_check(erc, "\0", "\0"); - festate->m_cursor[bIx] = - ix->Search(&festate->m_stateKey[bIx], false, false, festate->m_currTxn->GetThdId(), found); break; } for (int i = 0; i < 2; i++) { if (i == 1 && festate->m_bestIx->m_end < 0) { - if (festate->m_forwardDirectionScan) { - uint8_t* buf = nullptr; - MOT::Index* ix = festate->m_bestIx->m_ix; - uint16_t keyLength = ix->GetKeyLength(); - - festate->m_stateKey[1].InitKey(keyLength); - buf = festate->m_stateKey[1].GetKeyBuf(); - errno_t erc = memset_s(buf, keyLength, 0xff, keyLength); - securec_check(erc, "\0", "\0"); - festate->m_cursor[1] = - ix->Search(&festate->m_stateKey[1], false, false, festate->m_currTxn->GetThdId(), found); - } else { - festate->m_cursor[1] = festate->m_bestIx->m_ix->Begin(festate->m_currTxn->GetThdId()); - } + festate->m_cursor[1] = nullptr; break; } KEY_OPER oper = (i == 0 ? festate->m_bestIx->m_ixOpers[0] : festate->m_bestIx->m_ixOpers[1]); - forwardDirection = ((oper & ~KEY_OPER_PREFIX_BITMASK) < KEY_OPER::READ_KEY_OR_PREV); + forwardDirection = ((static_cast(oper) & ~KEY_OPER_PREFIX_BITMASK) < + static_cast(KEY_OPER::READ_KEY_OR_PREV)); CreateKeyBuffer(rel, festate, i); @@ -878,7 +724,7 @@ void MOTAdaptor::OpenCursor(Relation rel, MOTFdwStateSt* festate) break; default: - elog(INFO, "Invalid key operation: %u", oper); + elog(INFO, "Invalid key operation: %" PRIu8, static_cast(oper)); break; } @@ -893,6 +739,11 @@ void MOTAdaptor::OpenCursor(Relation rel, MOTFdwStateSt* festate) } } } while (0); + for (int i = 0; i < 2; i++) { + if (festate->m_cursor[i] != nullptr) { + festate->m_currTxn->m_queryState[(uint64_t)festate->m_cursor[i]] = (uint64_t)(festate->m_cursor[i]); + } + } } static void VarLenFieldType( @@ -913,7 +764,7 @@ static void VarLenFieldType( } /* fall through */ case 'e': -#ifdef USE_ASSERT_CHECKING +#ifdef MOT_SUPPORT_TEXT_FIELD if (typoid == TEXTOID) *typeLen = colLen = MAX_VARCHAR_LEN; #endif @@ -1023,7 +874,7 @@ static MOT::RC TableFieldType( return res; } -void MOTAdaptor::ValidateCreateIndex(IndexStmt* stmt, MOT::Table* table, MOT::TxnManager* txn) +static void ValidateCreateIndex(IndexStmt* stmt, MOT::Table* table, MOT::TxnManager* txn) { if (stmt->primary) { if (!table->IsTableEmpty(txn->GetThdId())) { @@ -1032,19 +883,16 @@ void MOTAdaptor::ValidateCreateIndex(IndexStmt* stmt, MOT::Table* table, MOT::Tx errcode(ERRCODE_FDW_ERROR), errmsg( "Table %s is not empty, create primary index is not allowed", table->GetTableName().c_str()))); - return; } - } else if (table->GetNumIndexes() == MAX_NUM_INDEXES) { + } else if (table->GetNumIndexes() >= MAX_NUM_INDEXES) { ereport(ERROR, (errmodule(MOD_MOT), errcode(ERRCODE_FDW_TOO_MANY_INDEXES), - errmsg("Can not create index, max number of indexes %u reached", MAX_NUM_INDEXES))); - return; + errmsg("Cannot create index, max number of indexes %u reached", MAX_NUM_INDEXES))); } if (strcmp(stmt->accessMethod, "btree") != 0) { ereport(ERROR, (errmodule(MOD_MOT), errmsg("MOT supports indexes of type BTREE only (btree or btree_art)"))); - return; } if (list_length(stmt->indexParams) > (int)MAX_KEY_COLUMNS) { @@ -1054,14 +902,13 @@ void MOTAdaptor::ValidateCreateIndex(IndexStmt* stmt, MOT::Table* table, MOT::Tx errmsg("Can't create index"), errdetail( "Number of columns exceeds %d max allowed %u", list_length(stmt->indexParams), MAX_KEY_COLUMNS))); - return; } } MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) { MOT::RC res; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); txn->SetTransactionId(tid); MOT::Table* table = txn->GetTableByExternalId(stmt->relation->foreignOid); @@ -1074,7 +921,18 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) return MOT::RC_ERROR; } - ValidateCreateIndex(stmt, table, txn); + table->GetOrigTable()->WrLock(); + + PG_TRY(); + { + ValidateCreateIndex(stmt, table, txn); + } + PG_CATCH(); + { + table->GetOrigTable()->Unlock(); + PG_RE_THROW(); + } + PG_END_TRY(); elog(LOG, "creating %s index %s (OID: %u), for table: %s", @@ -1082,7 +940,7 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) stmt->idxname, stmt->indexOid, stmt->relation->relname); - uint64_t keyLength = 0; + uint32_t keyLength = 0; MOT::Index* index = nullptr; MOT::IndexOrder index_order = MOT::IndexOrder::INDEX_ORDER_SECONDARY; @@ -1093,23 +951,45 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) // check if we have primary and delete previous definition if (stmt->primary) { index_order = MOT::IndexOrder::INDEX_ORDER_PRIMARY; + } else { + if (stmt->unique) { + index_order = MOT::IndexOrder::INDEX_ORDER_SECONDARY_UNIQUE; + } } index = MOT::IndexFactory::CreateIndex(index_order, indexing_method, flavor); if (index == nullptr) { + table->GetOrigTable()->Unlock(); report_pg_error(MOT::RC_ABORT); return MOT::RC_ABORT; } index->SetExtId(stmt->indexOid); - index->SetNumTableFields((uint32_t)table->GetFieldCount()); + if (!index->SetNumTableFields(table->GetFieldCount())) { + table->GetOrigTable()->Unlock(); + delete index; + report_pg_error(MOT::RC_ABORT); + return MOT::RC_ABORT; + } + int count = 0; ListCell* lc = nullptr; foreach (lc, stmt->indexParams) { IndexElem* ielem = (IndexElem*)lfirst(lc); + if (ielem->expr != nullptr) { + table->GetOrigTable()->Unlock(); + delete index; + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_INVALID_COLUMN_DEFINITION), + errmsg("Can't create index on field"), + errdetail("Expressions are not supported"))); + return MOT::RC_ERROR; + } uint64_t colid = table->GetFieldId((ielem->name != nullptr ? ielem->name : ielem->indexcolname)); if (colid == (uint64_t)-1) { // invalid column + table->GetOrigTable()->Unlock(); delete index; ereport(ERROR, (errmodule(MOD_MOT), @@ -1123,6 +1003,7 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) // Temp solution for NULLs, do not allow index creation on column that does not carry not null flag if (!MOT::GetGlobalConfiguration().m_allowIndexOnNullableColumn && !col->m_isNotNull) { + table->GetOrigTable()->Unlock(); delete index; ereport(ERROR, (errmodule(MOD_MOT), @@ -1134,6 +1015,7 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) // Temp solution, we have to support DECIMAL and NUMERIC indexes as well if (col->m_type == MOT::MOT_CATALOG_FIELD_TYPES::MOT_TYPE_DECIMAL) { + table->GetOrigTable()->Unlock(); delete index; ereport(ERROR, (errmodule(MOD_MOT), @@ -1143,6 +1025,7 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) return MOT::RC_ERROR; } if (col->m_keySize > MAX_KEY_SIZE) { + table->GetOrigTable()->Unlock(); delete index; ereport(ERROR, (errmodule(MOD_MOT), @@ -1160,29 +1043,83 @@ MOT::RC MOTAdaptor::CreateIndex(IndexStmt* stmt, ::TransactionId tid) index->SetNumIndexFields(count); if ((res = index->IndexInit(keyLength, stmt->unique, stmt->idxname, nullptr)) != MOT::RC_OK) { + table->GetOrigTable()->Unlock(); delete index; report_pg_error(res); return res; } res = txn->CreateIndex(table, index, stmt->primary); + if (res == MOT::RC_OK) { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + index->GetExtId(), MOT::DDL_ACCESS_CREATE_INDEX, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); + } + table->GetOrigTable()->Unlock(); if (res != MOT::RC_OK) { delete index; if (res == MOT::RC_TABLE_EXCEEDS_MAX_INDEXES) { ereport(ERROR, (errmodule(MOD_MOT), errcode(ERRCODE_FDW_TOO_MANY_INDEXES), - errmsg("Can not create index, max number of indexes %u reached", MAX_NUM_INDEXES))); + errmsg("Cannot create index, max number of indexes %u reached", MAX_NUM_INDEXES))); return MOT::RC_TABLE_EXCEEDS_MAX_INDEXES; - } else { - report_pg_error(txn->m_err, stmt->idxname, txn->m_errMsgBuf); - return MOT::RC_UNIQUE_VIOLATION; } + report_pg_error(txn->m_err, stmt->idxname, txn->m_errMsgBuf); + return MOT::RC_UNIQUE_VIOLATION; } return MOT::RC_OK; } +static void CalculateDecimalColumnTypeLen(MOT::MOT_CATALOG_FIELD_TYPES colType, ColumnDef* colDef, int16& typeLen) +{ + if (colType == MOT::MOT_CATALOG_FIELD_TYPES::MOT_TYPE_DECIMAL) { + if (list_length(colDef->typname->typmods) > 0) { + bool canMakeShort = true; + int precision = 0; + int scale = 0; + int count = 0; + + ListCell* c = nullptr; + foreach (c, colDef->typname->typmods) { + Node* d = (Node*)lfirst(c); + if (!IsA(d, A_Const)) { + canMakeShort = false; + break; + } + A_Const* ac = (A_Const*)d; + + if (ac->val.type != T_Integer) { + canMakeShort = false; + break; + } + + if (count == 0) { + precision = ac->val.val.ival; + } else { + scale = ac->val.val.ival; + } + + count++; + } + + if (canMakeShort) { + int len = 0; + + len += scale / DEC_DIGITS; + len += (scale % DEC_DIGITS > 0 ? 1 : 0); + + precision -= scale; + + len += precision / DEC_DIGITS; + len += (precision % DEC_DIGITS > 0 ? 1 : 0); + + typeLen = sizeof(MOT::DecimalSt) + len * sizeof(NumericDigit); + } + } + } +} + void MOTAdaptor::AddTableColumns(MOT::Table* table, List* tableElts, bool& hasBlob) { hasBlob = false; @@ -1214,51 +1151,7 @@ void MOTAdaptor::AddTableColumns(MOT::Table* table, List* tableElts, bool& hasBl } hasBlob |= isBlob; - if (colType == MOT::MOT_CATALOG_FIELD_TYPES::MOT_TYPE_DECIMAL) { - if (list_length(colDef->typname->typmods) > 0) { - bool canMakeShort = true; - int precision = 0; - int scale = 0; - int count = 0; - - ListCell* c = nullptr; - foreach (c, colDef->typname->typmods) { - Node* d = (Node*)lfirst(c); - if (!IsA(d, A_Const)) { - canMakeShort = false; - break; - } - A_Const* ac = (A_Const*)d; - - if (ac->val.type != T_Integer) { - canMakeShort = false; - break; - } - - if (count == 0) { - precision = ac->val.val.ival; - } else { - scale = ac->val.val.ival; - } - - count++; - } - - if (canMakeShort) { - int len = 0; - - len += scale / DEC_DIGITS; - len += (scale % DEC_DIGITS > 0 ? 1 : 0); - - precision -= scale; - - len += precision / DEC_DIGITS; - len += (precision % DEC_DIGITS > 0 ? 1 : 0); - - typeLen = sizeof(MOT::DecimalSt) + len * sizeof(NumericDigit); - } - } - } + CalculateDecimalColumnTypeLen(colType, colDef, typeLen); res = table->AddColumn(colDef->colname, typeLen, colType, colDef->is_not_null, typoid); if (res != MOT::RC_OK) { @@ -1269,12 +1162,37 @@ void MOTAdaptor::AddTableColumns(MOT::Table* table, List* tableElts, bool& hasBl } } } - +/* function name: CreateTable + function purpose:This function allows to create a corresponding table in MOT from a table defined from an external data source (opengauss). + This way, operations between the two systems can be synchronized and users can use them seamlessly. + + input: + CreateForeignTableStmt* stmt: + This is a pointer to the CreateForeignTableStmt structure that originates from GaussSQL's internal representation. It contains all detailed information about the foreign table to be created. This information includes: + Table name + Column information (such as column names, data types, etc.) + The table's OID (Object Identifier) + Possibly other metadata related to table attributes. + TransactionId tid: + This is a transaction ID, representing the currently executing GaussSQL transaction. This is necessary when initializing the MOT transaction manager, + as the MOT transaction needs to synchronize with the current GaussSQL transaction. + output: + The function's return value is of the type MOT::RC (MOT's Return Code), which is an enumeration indicating the result of the function's execution. For example: + MOT::RC_OK: Indicates the operation was successful. + MOT::RC_ERROR: Indicates a general error occurred. + MOT::RC_MEMORY_ALLOCATION_ERROR: Indicates there was an error trying to allocate memory. + Other MOT error codes may also be returned as a result. + If the function returns MOT::RC_OK, it means the MOT table was successfully created. Any other return value indicates some kind of problem was encountered while trying to create the table. + note:none + annotator:liushifa + annotate time:2023/08/12 12:35:34 + contact:3325287047@qq.com +*/ MOT::RC MOTAdaptor::CreateTable(CreateForeignTableStmt* stmt, ::TransactionId tid) { bool hasBlob = false; MOT::Index* primaryIdx = nullptr; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__, tid); MOT::Table* table = nullptr; MOT::RC res = MOT::RC_ERROR; @@ -1305,16 +1223,16 @@ MOT::RC MOTAdaptor::CreateTable(CreateForeignTableStmt* stmt, ::TransactionId ti errmsg("database with OID %u does not exist", u_sess->proc_cxt.MyDatabaseId))); break; } - tname.append(dbname); - tname.append("_"); + (void)tname.append(dbname); + (void)tname.append("_"); if (stmt->base.relation->schemaname != nullptr) { - tname.append(stmt->base.relation->schemaname); + (void)tname.append(stmt->base.relation->schemaname); } else { - tname.append("#"); + (void)tname.append("#"); } - tname.append("_"); - tname.append(stmt->base.relation->relname); + (void)tname.append("_"); + (void)tname.append(stmt->base.relation->relname); if (!table->Init(stmt->base.relation->relname, tname.c_str(), columnCount, stmt->base.relation->foreignOid)) { delete table; @@ -1362,6 +1280,13 @@ MOT::RC MOTAdaptor::CreateTable(CreateForeignTableStmt* stmt, ::TransactionId ti break; } + if (!table->InitTombStonePool()) { + delete table; + table = nullptr; + report_pg_error(MOT::RC_MEMORY_ALLOCATION_ERROR); + break; + } + elog(LOG, "creating table %s (OID: %u), num columns: %u, tuple: %u", table->GetLongTableName().c_str(), @@ -1381,12 +1306,19 @@ MOT::RC MOTAdaptor::CreateTable(CreateForeignTableStmt* stmt, ::TransactionId ti primaryIdx = MOT::IndexFactory::CreatePrimaryIndexEx( MOT::IndexingMethod::INDEXING_METHOD_TREE, DEFAULT_TREE_FLAVOR, 8, table->GetLongTableName(), res, nullptr); if (res != MOT::RC_OK) { - txn->DropTable(table); + (void)txn->DropTable(table); report_pg_error(res); break; } primaryIdx->SetExtId(stmt->base.relation->foreignOid + 1); - primaryIdx->SetNumTableFields(columnCount); + if (!primaryIdx->SetNumTableFields(columnCount)) { + res = MOT::RC_MEMORY_ALLOCATION_ERROR; + (void)txn->DropTable(table); + delete primaryIdx; + report_pg_error(res); + break; + } + primaryIdx->SetNumIndexFields(1); primaryIdx->SetLenghtKeyFields(0, -1, 8); primaryIdx->SetFakePrimary(true); @@ -1397,44 +1329,78 @@ MOT::RC MOTAdaptor::CreateTable(CreateForeignTableStmt* stmt, ::TransactionId ti if (res != MOT::RC_OK) { if (table != nullptr) { - txn->DropTable(table); + (void)txn->DropTable(table); } if (primaryIdx != nullptr) { delete primaryIdx; } + } else { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + table->GetTableExId(), MOT::DDL_ACCESS_CREATE_TABLE, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); } return res; } - +/* function name: DropIndex + function purpose: Used to drop indexes in the mot engine. + input: Same as function input above + output: Same as function output above + note: Dropping primary index is not supported. + annotator:liushifa + annotate time:2023/08/12 17:47:04 + contact:3325287047@qq.com +*/ MOT::RC MOTAdaptor::DropIndex(DropForeignStmt* stmt, ::TransactionId tid) -{ +{ + //Initialize the return value to OK. MOT::RC res = MOT::RC_OK; - EnsureSafeThreadAccessInline(); + // Ensure the current thread has safe access to MOT. + (void)EnsureSafeThreadAccessInline(); + // Initialize the MOT transaction manager for the current operation MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); txn->SetTransactionId(tid); elog(LOG, "dropping index %s, ixoid: %u, taboid: %u", stmt->name, stmt->indexoid, stmt->reloid); - // get table + // Drop the index. do { - MOT::Index* index = txn->GetIndexByExternalId(stmt->reloid, stmt->indexoid); + // Retrieve the table associated with the index being dropped. + MOT::Table* table = txn->GetTableByExternalId(stmt->reloid); + // Check if the table exists. + if (table == nullptr) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("Table not found for oid %u", stmt->reloid))); + return MOT::RC_ERROR; + } + // Lock the table to ensure safe operation. + table->GetOrigTable()->WrLock(); + // Find the index in the table by its external OID. + MOT::Index* index = table->GetIndexByExtId(stmt->indexoid); + // Check if the index exists. if (index == nullptr) { + table->GetOrigTable()->Unlock(); elog(LOG, "Drop index %s error, index oid %u of table oid %u not found.", stmt->name, stmt->indexoid, stmt->reloid); res = MOT::RC_INDEX_NOT_FOUND; - } else if (index->IsPrimaryKey()) { + } + // Check if the index is a primary key index. Dropping primary index is not supported. + else if (index->IsPrimaryKey()) { + table->GetOrigTable()->Unlock(); elog(LOG, "Drop primary index is not supported, failed to drop index: %s", stmt->name); - } else { - MOT::Table* table = index->GetTable(); - uint64_t table_relid = table->GetTableExId(); - JitExec::PurgeJitSourceCache(table_relid, false); - table->WrLock(); + } + // Drop the index using the transaction manager. + else { res = txn->DropIndex(index); - table->Unlock(); + if (res == MOT::RC_OK) { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + table->GetTableExId(), MOT::DDL_ACCESS_DROP_INDEX, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); + } + table->GetOrigTable()->Unlock(); } } while (0); @@ -1455,21 +1421,247 @@ MOT::RC MOTAdaptor::DropTable(DropForeignStmt* stmt, ::TransactionId tid) res = MOT::RC_TABLE_NOT_FOUND; elog(LOG, "Drop table %s error, table oid %u not found.", stmt->name, stmt->reloid); } else { - uint64_t table_relid = tab->GetTableExId(); - JitExec::PurgeJitSourceCache(table_relid, false); + tab->GetOrigTable()->WrLock(); res = txn->DropTable(tab); + if (res == MOT::RC_OK) { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + tab->GetTableExId(), MOT::DDL_ACCESS_DROP_TABLE, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); + } + tab->GetOrigTable()->Unlock(); } } while (0); return res; } +MOT::RC MOTAdaptor::AlterTableAddColumn(AlterForeingTableCmd* cmd, TransactionId tid) +{ + MOT::RC res = MOT::RC_OK; + MOT::Table* tab = nullptr; + MOT::Column* newColumn = nullptr; + int16 typeLen = 0; + ColumnDef* colDef = (ColumnDef*)cmd->def; + (void)EnsureSafeThreadAccessInline(); + + MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); + txn->SetTransactionId(tid); + do { + bool isBlob = false; + Oid typoid = InvalidOid; + MOT::MOT_CATALOG_FIELD_TYPES colType; + tab = txn->GetTableByExternalId(cmd->rel->rd_id); + if (tab == nullptr) { + elog(LOG, + "Alter table add column %s error, table oid %u not found.", + NameStr(cmd->rel->rd_rel->relname), + cmd->rel->rd_id); + break; + } + + res = TableFieldType(colDef, colType, &typeLen, typoid, isBlob); + if (res != MOT::RC_OK) { + report_pg_error(res, colDef, (void*)(int64)typeLen); + break; + } + + CalculateDecimalColumnTypeLen(colType, colDef, typeLen); + + // check default value + size_t dftSize = 0; + uintptr_t dftSrc = 0; + Datum dftValue = 0; + bytea* txt = nullptr; + bool shouldFreeTxt = false; + bool isNull = false; + bool hasDefault = false; + char buf[DECIMAL_MAX_SIZE]; + MOT::DecimalSt* d = (MOT::DecimalSt*)buf; + if (cmd->defValue != nullptr) { + if (contain_volatile_functions((Node*)cmd->defValue)) { + ereport(ERROR, + (errcode(ERRCODE_FDW_OPERATION_NOT_SUPPORTED), + errmodule(MOD_MOT), + errmsg("Add column does not support volatile default value"))); + break; + } + + EState* estate = CreateExecutorState(); + ExprState* exprstate = ExecInitExpr(expression_planner(cmd->defValue), NULL); + ExprContext* econtext = GetPerTupleExprContext(estate); + + MemoryContext newcxt = GetPerTupleMemoryContext(estate); + MemoryContext oldcxt = MemoryContextSwitchTo(newcxt); + dftValue = ExecEvalExpr(exprstate, econtext, &isNull, NULL); + (void)MemoryContextSwitchTo(oldcxt); + if (!isNull) { + hasDefault = true; + switch (exprstate->resultType) { + case BYTEAOID: + case TEXTOID: + case VARCHAROID: + case CLOBOID: + case BPCHAROID: { + txt = DatumGetByteaP(dftValue); + dftSize = VARSIZE(txt) - VARHDRSZ; // includes header len VARHDRSZ + dftSrc = (uintptr_t)VARDATA(txt); + shouldFreeTxt = true; + break; + } + case NUMERICOID: { + Numeric n = DatumGetNumeric(dftValue); + if (NUMERIC_NDIGITS(n) > DECIMAL_MAX_DIGITS) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("Value exceeds maximum precision: %d", NUMERIC_MAX_PRECISION))); + break; + } + PGNumericToMOT(n, *d); + dftSize = DECIMAL_SIZE(d); + dftSrc = (uintptr_t)d; + break; + } + default: + dftSize = typeLen; + dftSrc = (uintptr_t)dftValue; + break; + } + } + FreeExecutorState(estate); + } + tab->GetOrigTable()->WrLock(); + res = tab->CreateColumn( + newColumn, colDef->colname, typeLen, colType, colDef->is_not_null, typoid, hasDefault, dftSrc, dftSize); + if (res != MOT::RC_OK) { + tab->GetOrigTable()->Unlock(); + report_pg_error(res, colDef, (void*)(int64)typeLen); + break; + } + if (newColumn != nullptr) { + res = txn->AlterTableAddColumn(tab, newColumn); + if (res == MOT::RC_OK) { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + tab->GetTableExId(), MOT::DDL_ACCESS_ADD_COLUMN, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); + } + } + tab->GetOrigTable()->Unlock(); + // free if allocated + if (shouldFreeTxt && (char*)dftSrc != (char*)txt) { + pfree(txt); + } + } while (false); + if (res != MOT::RC_OK) { + if (newColumn != nullptr) { + delete newColumn; + } + report_pg_error(res, colDef, (void*)tab); + } + return res; +} + +MOT::RC MOTAdaptor::AlterTableDropColumn(AlterForeingTableCmd* cmd, TransactionId tid) +{ + MOT::RC res = MOT::RC_OK; + (void)EnsureSafeThreadAccessInline(); + + MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); + txn->SetTransactionId(tid); + do { + MOT::Table* tab = txn->GetTableByExternalId(cmd->rel->rd_id); + if (tab == nullptr) { + elog(LOG, + "Alter table add column %s error, table oid %u not found.", + NameStr(cmd->rel->rd_rel->relname), + cmd->rel->rd_id); + break; + } + tab->GetOrigTable()->WrLock(); + uint64_t colId = tab->GetFieldId(cmd->name); + if (colId == (uint64_t)-1) { + tab->GetOrigTable()->Unlock(); + ereport(ERROR, + (errcode(ERRCODE_FDW_COLUMN_NAME_NOT_FOUND), + errmodule(MOD_MOT), + errmsg("Column %s not found", cmd->name))); + break; + } + MOT::Column* col = tab->GetField(colId); + if (col->IsUsedByIndex()) { + tab->GetOrigTable()->Unlock(); + ereport(ERROR, + (errcode(ERRCODE_FDW_DROP_INDEXED_COLUMN_NOT_ALLOWED), + errmodule(MOD_MOT), + errmsg("Drop column %s is not allowed, used by one or more indexes", cmd->name))); + break; + } + res = txn->AlterTableDropColumn(tab, col); + if (res == MOT::RC_OK) { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + tab->GetTableExId(), MOT::DDL_ACCESS_DROP_COLUMN, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); + } + tab->GetOrigTable()->Unlock(); + } while (false); + if (res != MOT::RC_OK) { + report_pg_error(res); + } + return res; +} + +MOT::RC MOTAdaptor::AlterTableRenameColumn(RenameForeingTableCmd* cmd, TransactionId tid) +{ + MOT::RC res = MOT::RC_OK; + (void)EnsureSafeThreadAccessInline(); + + MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); + txn->SetTransactionId(tid); + do { + MOT::Table* tab = txn->GetTableByExternalId(cmd->relid); + if (tab == nullptr) { + elog(LOG, "Alter table rename column error, table oid %u not found.", cmd->relid); + break; + } + tab->GetOrigTable()->WrLock(); + uint64_t colId = tab->GetFieldId(cmd->oldname); + if (colId == (uint64_t)-1) { + tab->GetOrigTable()->Unlock(); + ereport(ERROR, + (errcode(ERRCODE_FDW_COLUMN_NAME_NOT_FOUND), + errmodule(MOD_MOT), + errmsg("Column %s not found", cmd->oldname))); + break; + } + uint16_t len = strlen(cmd->newname); + if (len >= MOT::Column::MAX_COLUMN_NAME_LEN) { + tab->GetOrigTable()->Unlock(); + ereport(ERROR, + (errcode(ERRCODE_INVALID_COLUMN_DEFINITION), + errmodule(MOD_MOT), + errmsg("Column definition of %s is not supported", cmd->newname), + errdetail("Column name %s exceeds max name size %u", + cmd->newname, + (uint32_t)MOT::Column::MAX_COLUMN_NAME_LEN))); + break; + } + MOT::Column* col = tab->GetField(colId); + res = txn->AlterTableRenameColumn(tab, col, cmd->newname); + if (res == MOT::RC_OK) { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + tab->GetTableExId(), MOT::DDL_ACCESS_RENAME_COLUMN, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); + } + tab->GetOrigTable()->Unlock(); + } while (false); + if (res != MOT::RC_OK) { + report_pg_error(res); + } + return res; +} + MOT::RC MOTAdaptor::TruncateTable(Relation rel, ::TransactionId tid) { MOT::RC res = MOT::RC_OK; MOT::Table* tab = nullptr; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); txn->SetTransactionId(tid); @@ -1482,26 +1674,28 @@ MOT::RC MOTAdaptor::TruncateTable(Relation rel, ::TransactionId tid) break; } - JitExec::PurgeJitSourceCache(rel->rd_id, true); - tab->WrLock(); + tab->GetOrigTable()->WrLock(); res = txn->TruncateTable(tab); - tab->Unlock(); + if (res == MOT::RC_OK) { + MOT::MOTEngine::GetInstance()->NotifyDDLEvent( + tab->GetTableExId(), MOT::DDL_ACCESS_TRUNCATE_TABLE, MOT::TxnDDLPhase::TXN_DDL_PHASE_EXEC); + } + tab->GetOrigTable()->Unlock(); } while (0); return res; } -MOT::RC MOTAdaptor::VacuumTable(Relation rel, ::TransactionId tid) +void MOTAdaptor::VacuumTable(Relation rel, ::TransactionId tid) { - MOT::RC res = MOT::RC_OK; MOT::Table* tab = nullptr; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); txn->SetTransactionId(tid); elog(LOG, "vacuuming table %s, oid: %u", NameStr(rel->rd_rel->relname), rel->rd_id); do { - tab = MOT::GetTableManager()->GetTableSafeByExId(rel->rd_id); + tab = MOT::GetTableManager()->GetTableSafeByExId(rel->rd_id, true); if (tab == nullptr) { elog(LOG, "Vacuum table %s error, table oid %u not found.", NameStr(rel->rd_rel->relname), rel->rd_id); break; @@ -1510,16 +1704,16 @@ MOT::RC MOTAdaptor::VacuumTable(Relation rel, ::TransactionId tid) tab->Compact(txn); tab->Unlock(); } while (0); - return res; } uint64_t MOTAdaptor::GetTableIndexSize(uint64_t tabId, uint64_t ixId) { uint64_t res = 0; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MOT::TxnManager* txn = GetSafeTxn(__FUNCTION__); MOT::Table* tab = nullptr; MOT::Index* ix = nullptr; + uint64_t netTotal = 0; do { tab = txn->GetTableByExternalId(tabId); @@ -1540,9 +1734,10 @@ uint64_t MOTAdaptor::GetTableIndexSize(uint64_t tabId, uint64_t ixId) errmsg("Get index size error, index oid %lu for table oid %lu not found.", ixId, tabId))); break; } - res = ix->GetIndexSize(); - } else - res = tab->GetTableSize(); + res = ix->GetIndexSize(netTotal); + } else { + res = tab->GetTableSize(netTotal); + } } while (0); return res; @@ -1550,21 +1745,15 @@ uint64_t MOTAdaptor::GetTableIndexSize(uint64_t tabId, uint64_t ixId) MotMemoryDetail* MOTAdaptor::GetMemSize(uint32_t* nodeCount, bool isGlobal) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MotMemoryDetail* result = nullptr; *nodeCount = 0; /* We allocate an array of size (m_nodeCount + 1) to accommodate one aggregated entry of all global pools. */ uint32_t statsArraySize = MOT::g_memGlobalCfg.m_nodeCount + 1; MOT::MemRawChunkPoolStats* chunkPoolStatsArray = - (MOT::MemRawChunkPoolStats*)palloc(statsArraySize * sizeof(MOT::MemRawChunkPoolStats)); + (MOT::MemRawChunkPoolStats*)palloc0(statsArraySize * sizeof(MOT::MemRawChunkPoolStats)); if (chunkPoolStatsArray != nullptr) { - errno_t erc = memset_s(chunkPoolStatsArray, - statsArraySize * sizeof(MOT::MemRawChunkPoolStats), - 0, - statsArraySize * sizeof(MOT::MemRawChunkPoolStats)); - securec_check(erc, "\0", "\0"); - uint32_t realStatsEntries; if (isGlobal) { realStatsEntries = MOT::MemRawChunkStoreGetGlobalStats(chunkPoolStatsArray, statsArraySize); @@ -1574,7 +1763,7 @@ MotMemoryDetail* MOTAdaptor::GetMemSize(uint32_t* nodeCount, bool isGlobal) MOT_ASSERT(realStatsEntries <= statsArraySize); if (realStatsEntries > 0) { - result = (MotMemoryDetail*)palloc(realStatsEntries * sizeof(MotMemoryDetail)); + result = (MotMemoryDetail*)palloc0(realStatsEntries * sizeof(MotMemoryDetail)); if (result != nullptr) { for (uint32_t node = 0; node < realStatsEntries; ++node) { result[node].numaNode = chunkPoolStatsArray[node].m_node; @@ -1592,17 +1781,17 @@ MotMemoryDetail* MOTAdaptor::GetMemSize(uint32_t* nodeCount, bool isGlobal) MotSessionMemoryDetail* MOTAdaptor::GetSessionMemSize(uint32_t* sessionCount) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); MotSessionMemoryDetail* result = nullptr; *sessionCount = 0; uint32_t session_count = MOT::g_memGlobalCfg.m_maxThreadCount; MOT::MemSessionAllocatorStats* session_stats_array = - (MOT::MemSessionAllocatorStats*)palloc(session_count * sizeof(MOT::MemSessionAllocatorStats)); + (MOT::MemSessionAllocatorStats*)palloc0(session_count * sizeof(MOT::MemSessionAllocatorStats)); if (session_stats_array != nullptr) { uint32_t real_session_count = MOT::MemSessionGetAllStats(session_stats_array, session_count); if (real_session_count > 0) { - result = (MotSessionMemoryDetail*)palloc(real_session_count * sizeof(MotSessionMemoryDetail)); + result = (MotSessionMemoryDetail*)palloc0(real_session_count * sizeof(MotSessionMemoryDetail)); if (result != nullptr) { for (uint32_t session_index = 0; session_index < real_session_count; ++session_index) { GetSessionDetails(session_stats_array[session_index].m_sessionId, @@ -1625,7 +1814,7 @@ void MOTAdaptor::CreateKeyBuffer(Relation rel, MOTFdwStateSt* festate, int start { uint8_t* buf = nullptr; uint8_t pattern = 0x00; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); int16_t num = festate->m_bestIx->m_ix->GetNumFields(); const uint16_t* fieldLengths = festate->m_bestIx->m_ix->GetLengthKeyFields(); const int16_t* orgCols = festate->m_bestIx->m_ix->GetColumnKeyFields(); @@ -1671,7 +1860,7 @@ void MOTAdaptor::CreateKeyBuffer(Relation rel, MOTFdwStateSt* festate, int start break; default: - elog(LOG, "Invalid key operation: %u", oper); + elog(LOG, "Invalid key operation: %" PRIu8, static_cast(oper)); break; } @@ -1679,7 +1868,7 @@ void MOTAdaptor::CreateKeyBuffer(Relation rel, MOTFdwStateSt* festate, int start if (opers[i] < KEY_OPER::READ_INVALID) { bool is_null = false; ExprState* expr = (ExprState*)list_nth(festate->m_execExprs, exprs[i] - 1); - Datum val = ExecEvalExpr((ExprState*)(expr), festate->m_econtext, &is_null, nullptr); + Datum val = ExecEvalExpr((ExprState*)(expr), festate->m_econtext, &is_null); if (is_null) { MOT_ASSERT((offset + fieldLengths[i]) <= keyLength); errno_t erc = memset_s(buf + offset, fieldLengths[i], 0x00, fieldLengths[i]); @@ -1709,7 +1898,7 @@ void MOTAdaptor::CreateKeyBuffer(Relation rel, MOTFdwStateSt* festate, int start case KEY_OPER::READ_KEY_EXACT: default: - elog(LOG, "Invalid key operation: %u", oper); + elog(LOG, "Invalid key operation: %" PRIu8, static_cast(oper)); break; } } @@ -1717,7 +1906,7 @@ void MOTAdaptor::CreateKeyBuffer(Relation rel, MOTFdwStateSt* festate, int start DatumToMOTKey(col, expr->resultType, val, - desc->attrs[orgCols[i] - 1]->atttypid, + desc->attrs[orgCols[i] - 1].atttypid, buf + offset, fieldLengths[i], opers[i], @@ -1725,7 +1914,7 @@ void MOTAdaptor::CreateKeyBuffer(Relation rel, MOTFdwStateSt* festate, int start } } else { MOT_ASSERT((offset + fieldLengths[i]) <= keyLength); - festate->m_stateKey[start].FillPattern(pattern, fieldLengths[i], offset); + (void)festate->m_stateKey[start].FillPattern(pattern, fieldLengths[i], offset); } offset += fieldLengths[i]; @@ -1737,7 +1926,7 @@ void MOTAdaptor::CreateKeyBuffer(Relation rel, MOTFdwStateSt* festate, int start bool MOTAdaptor::IsScanEnd(MOTFdwStateSt* festate) { bool res = false; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); // festate->cursor[1] (end iterator) might be NULL (in case it is not in use). If this is the case, return false // (which means we have not reached the end yet) @@ -1773,7 +1962,7 @@ bool MOTAdaptor::IsScanEnd(MOTFdwStateSt* festate) void MOTAdaptor::PackRow(TupleTableSlot* slot, MOT::Table* table, uint8_t* attrs_used, uint8_t* destRow) { errno_t erc; - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); HeapTuple srcData = (HeapTuple)slot->tts_tuple; TupleDesc tupdesc = slot->tts_tupleDescriptor; bool hasnulls = HeapTupleHasNulls(srcData); @@ -1799,14 +1988,14 @@ void MOTAdaptor::PackRow(TupleTableSlot* slot, MOT::Table* table, uint8_t* attrs Datum value = heap_slot_getattr(slot, j, &isnull); if (!isnull) { - DatumToMOT(table->GetField(j), value, tupdesc->attrs[i]->atttypid, destRow); + DatumToMOT(table->GetField(j), value, tupdesc->attrs[i].atttypid, destRow); } } } void MOTAdaptor::PackUpdateRow(TupleTableSlot* slot, MOT::Table* table, const uint8_t* attrs_used, uint8_t* destRow) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); TupleDesc tupdesc = slot->tts_tupleDescriptor; uint8_t* bits; uint64_t i = 0; @@ -1822,7 +2011,7 @@ void MOTAdaptor::PackUpdateRow(TupleTableSlot* slot, MOT::Table* table, const ui Datum value = heap_slot_getattr(slot, j, &isnull); if (!isnull) { - DatumToMOT(table->GetField(j), value, tupdesc->attrs[i]->atttypid, destRow); + DatumToMOT(table->GetField(j), value, tupdesc->attrs[i].atttypid, destRow); BITMAP_SET(bits, i); } else { BITMAP_CLEAR(bits, i); @@ -1833,7 +2022,7 @@ void MOTAdaptor::PackUpdateRow(TupleTableSlot* slot, MOT::Table* table, const ui void MOTAdaptor::UnpackRow(TupleTableSlot* slot, MOT::Table* table, const uint8_t* attrs_used, uint8_t* srcRow) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); TupleDesc tupdesc = slot->tts_tupleDescriptor; uint64_t i = 0; @@ -1841,9 +2030,9 @@ void MOTAdaptor::UnpackRow(TupleTableSlot* slot, MOT::Table* table, const uint8_ uint64_t cols = table->GetFieldCount() - 1; for (; i < cols; i++) { - if (BITMAP_GET(attrs_used, i)) - MOTToDatum(table, tupdesc->attrs[i], srcRow, &(slot->tts_values[i]), &(slot->tts_isnull[i])); - else { + if (BITMAP_GET(attrs_used, i)) { + MOTToDatum(table, &tupdesc->attrs[i], srcRow, &(slot->tts_values[i]), &(slot->tts_isnull[i])); + } else { slot->tts_isnull[i] = true; slot->tts_values[i] = PointerGetDatum(nullptr); } @@ -1853,7 +2042,7 @@ void MOTAdaptor::UnpackRow(TupleTableSlot* slot, MOT::Table* table, const uint8_ // useful functions for data conversion: utils/fmgr/gmgr.cpp void MOTAdaptor::MOTToDatum(MOT::Table* table, const Form_pg_attribute attr, uint8_t* data, Datum* value, bool* is_null) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); if (!BITMAP_GET(data, (attr->attnum - 1))) { *is_null = true; *value = PointerGetDatum(nullptr); @@ -1875,8 +2064,10 @@ void MOTAdaptor::MOTToDatum(MOT::Table* table, const Form_pg_attribute attr, uin col->Unpack(data, &tmp, len); bytea* result = (bytea*)palloc(len + VARHDRSZ); - errno_t erc = memcpy_s(VARDATA(result), len, (uint8_t*)tmp, len); - securec_check(erc, "\0", "\0"); + if (len > 0) { + errno_t erc = memcpy_s(VARDATA(result), len, (uint8_t*)tmp, len); + securec_check(erc, "\0", "\0"); + } SET_VARSIZE(result, len + VARHDRSZ); *value = PointerGetDatum(result); @@ -1897,7 +2088,7 @@ void MOTAdaptor::MOTToDatum(MOT::Table* table, const Form_pg_attribute attr, uin void MOTAdaptor::DatumToMOT(MOT::Column* col, Datum datum, Oid type, uint8_t* data) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); switch (type) { case BYTEAOID: case TEXTOID: @@ -1907,7 +2098,12 @@ void MOTAdaptor::DatumToMOT(MOT::Column* col, Datum datum, Oid type, uint8_t* da bytea* txt = DatumGetByteaP(datum); size_t size = VARSIZE(txt); // includes header len VARHDRSZ char* src = VARDATA(txt); - col->Pack(data, (uintptr_t)src, size - VARHDRSZ); + if (!col->Pack(data, (uintptr_t)src, size - VARHDRSZ)) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_STRING_DATA_RIGHT_TRUNCATION), + errmsg("value too long for column size (%d)", (int)(col->m_size - VARHDRSZ)))); + } if ((char*)datum != (char*)txt) { pfree(txt); @@ -1928,17 +2124,16 @@ void MOTAdaptor::DatumToMOT(MOT::Column* col, Datum datum, Oid type, uint8_t* da break; } PGNumericToMOT(n, *d); - col->Pack(data, (uintptr_t)d, DECIMAL_SIZE(d)); - + (void)col->Pack(data, (uintptr_t)d, DECIMAL_SIZE(d)); // simple types packing cannot fail break; } default: - col->Pack(data, datum, col->m_size); + (void)col->Pack(data, datum, col->m_size); // no verification required break; } } -inline void MOTAdaptor::VarcharToMOTKey( +void MOTAdaptor::VarcharToMOTKey( MOT::Column* col, Oid datumType, Datum datum, Oid colType, uint8_t* data, size_t len, KEY_OPER oper, uint8_t fill) { bool noValue = false; @@ -1963,11 +2158,6 @@ inline void MOTAdaptor::VarcharToMOTKey( bytea* txt = DatumGetByteaP(datum); size_t size = VARSIZE(txt); // includes header len VARHDRSZ char* src = VARDATA(txt); - - if (size > len) { - size = len; - } - size -= VARHDRSZ; if (oper == KEY_OPER::READ_KEY_LIKE) { if (src[size - 1] == '%') { @@ -1983,14 +2173,20 @@ inline void MOTAdaptor::VarcharToMOTKey( } else if (colType == BPCHAROID) { // handle padding for blank-padded type fill = 0x20; } - col->PackKey(data, (uintptr_t)src, size, fill); + // if the length of search value is bigger than a column key size (len) + // we make key only from a part that equals column key length + // this will override the column delimiter exactly by one char from search value + if (size > len) { + size = len; + } + (void)col->PackKey(data, (uintptr_t)src, size, fill); if ((char*)datum != (char*)txt) { pfree(txt); } } -inline void MOTAdaptor::FloatToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) +void MOTAdaptor::FloatToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) { if (datumType == FLOAT8OID) { MOT::DoubleConvT dc; @@ -1998,64 +2194,64 @@ inline void MOTAdaptor::FloatToMOTKey(MOT::Column* col, Oid datumType, Datum dat dc.m_r = (uint64_t)datum; fc.m_v = (float)dc.m_v; uint64_t u = (uint64_t)fc.m_r; - col->PackKey(data, u, col->m_size); + (void)col->PackKey(data, u, col->m_size); } else { - col->PackKey(data, datum, col->m_size); + (void)col->PackKey(data, datum, col->m_size); } } -inline void MOTAdaptor::NumericToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) +void MOTAdaptor::NumericToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) { Numeric n = DatumGetNumeric(datum); char buf[DECIMAL_MAX_SIZE]; MOT::DecimalSt* d = (MOT::DecimalSt*)buf; PGNumericToMOT(n, *d); - col->PackKey(data, (uintptr_t)d, DECIMAL_SIZE(d)); + (void)col->PackKey(data, (uintptr_t)d, DECIMAL_SIZE(d)); } -inline void MOTAdaptor::TimestampToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) +void MOTAdaptor::TimestampToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) { if (datumType == TIMESTAMPTZOID) { Timestamp result = DatumGetTimestamp(DirectFunctionCall1(timestamptz_timestamp, datum)); - col->PackKey(data, result, col->m_size); + (void)col->PackKey(data, result, col->m_size); } else if (datumType == DATEOID) { Timestamp result = DatumGetTimestamp(DirectFunctionCall1(date_timestamp, datum)); - col->PackKey(data, result, col->m_size); + (void)col->PackKey(data, result, col->m_size); } else { - col->PackKey(data, datum, col->m_size); + (void)col->PackKey(data, datum, col->m_size); } } -inline void MOTAdaptor::TimestampTzToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) +void MOTAdaptor::TimestampTzToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) { if (datumType == TIMESTAMPOID) { TimestampTz result = DatumGetTimestampTz(DirectFunctionCall1(timestamp_timestamptz, datum)); - col->PackKey(data, result, col->m_size); + (void)col->PackKey(data, result, col->m_size); } else if (datumType == DATEOID) { TimestampTz result = DatumGetTimestampTz(DirectFunctionCall1(date_timestamptz, datum)); - col->PackKey(data, result, col->m_size); + (void)col->PackKey(data, result, col->m_size); } else { - col->PackKey(data, datum, col->m_size); + (void)col->PackKey(data, datum, col->m_size); } } -inline void MOTAdaptor::DateToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) +void MOTAdaptor::DateToMOTKey(MOT::Column* col, Oid datumType, Datum datum, uint8_t* data) { if (datumType == TIMESTAMPOID) { DateADT result = DatumGetDateADT(DirectFunctionCall1(timestamp_date, datum)); - col->PackKey(data, result, col->m_size); + (void)col->PackKey(data, result, col->m_size); } else if (datumType == TIMESTAMPTZOID) { DateADT result = DatumGetDateADT(DirectFunctionCall1(timestamptz_date, datum)); - col->PackKey(data, result, col->m_size); + (void)col->PackKey(data, result, col->m_size); } else { - col->PackKey(data, datum, col->m_size); + (void)col->PackKey(data, datum, col->m_size); } } void MOTAdaptor::DatumToMOTKey( MOT::Column* col, Oid datumType, Datum datum, Oid colType, uint8_t* data, size_t len, KEY_OPER oper, uint8_t fill) { - EnsureSafeThreadAccessInline(); + (void)EnsureSafeThreadAccessInline(); switch (colType) { case BYTEAOID: case TEXTOID: @@ -2080,116 +2276,7 @@ void MOTAdaptor::DatumToMOTKey( DateToMOTKey(col, datumType, datum, data); break; default: - col->PackKey(data, datum, col->m_size); + (void)col->PackKey(data, datum, col->m_size); break; } } - -MOTFdwStateSt* InitializeFdwState(void* fdwState, List** fdwExpr, uint64_t exTableID) -{ - MOTFdwStateSt* state = (MOTFdwStateSt*)palloc0(sizeof(MOTFdwStateSt)); - List* values = (List*)fdwState; - - state->m_allocInScan = true; - state->m_foreignTableId = exTableID; - if (list_length(values) > 0) { - ListCell* cell = list_head(values); - int type = ((Const*)lfirst(cell))->constvalue; - if (type != FDW_LIST_STATE) { - return state; - } - cell = lnext(cell); - state->m_cmdOper = (CmdType)((Const*)lfirst(cell))->constvalue; - cell = lnext(cell); - state->m_order = (SORTDIR_ENUM)((Const*)lfirst(cell))->constvalue; - cell = lnext(cell); - state->m_hasForUpdate = (bool)((Const*)lfirst(cell))->constvalue; - cell = lnext(cell); - state->m_foreignTableId = ((Const*)lfirst(cell))->constvalue; - cell = lnext(cell); - state->m_numAttrs = ((Const*)lfirst(cell))->constvalue; - cell = lnext(cell); - state->m_ctidNum = ((Const*)lfirst(cell))->constvalue; - cell = lnext(cell); - state->m_numExpr = ((Const*)lfirst(cell))->constvalue; - cell = lnext(cell); - - int len = BITMAP_GETLEN(state->m_numAttrs); - state->m_attrsUsed = (uint8_t*)palloc0(len); - state->m_attrsModified = (uint8_t*)palloc0(len); - BitmapDeSerialize(state->m_attrsUsed, len, &cell); - - if (cell != NULL) { - state->m_bestIx = &state->m_bestIxBuf; - state->m_bestIx->Deserialize(cell, exTableID); - } - - if (fdwExpr != NULL && *fdwExpr != NULL) { - ListCell* c = NULL; - int i = 0; - - // divide fdw expr to param list and original expr - state->m_remoteCondsOrig = NULL; - - foreach (c, *fdwExpr) { - if (i < state->m_numExpr) { - i++; - continue; - } else { - state->m_remoteCondsOrig = lappend(state->m_remoteCondsOrig, lfirst(c)); - } - } - - *fdwExpr = list_truncate(*fdwExpr, state->m_numExpr); - } - } - return state; -} - -void* SerializeFdwState(MOTFdwStateSt* state) -{ - List* result = NULL; - - // set list type to FDW_LIST_STATE - result = lappend(result, makeConst(INT4OID, -1, InvalidOid, 4, FDW_LIST_STATE, false, true)); - result = lappend(result, makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(state->m_cmdOper), false, true)); - result = lappend(result, makeConst(INT1OID, -1, InvalidOid, 4, Int8GetDatum(state->m_order), false, true)); - result = lappend(result, makeConst(BOOLOID, -1, InvalidOid, 1, BoolGetDatum(state->m_hasForUpdate), false, true)); - result = - lappend(result, makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(state->m_foreignTableId), false, true)); - result = lappend(result, makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(state->m_numAttrs), false, true)); - result = lappend(result, makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(state->m_ctidNum), false, true)); - result = lappend(result, makeConst(INT2OID, -1, InvalidOid, 2, Int16GetDatum(state->m_numExpr), false, true)); - int len = BITMAP_GETLEN(state->m_numAttrs); - result = BitmapSerialize(result, state->m_attrsUsed, len); - - if (state->m_bestIx != nullptr) { - state->m_bestIx->Serialize(&result); - } - ReleaseFdwState(state); - return result; -} - -void ReleaseFdwState(MOTFdwStateSt* state) -{ - CleanCursors(state); - - if (state->m_currTxn) { - state->m_currTxn->m_queryState.erase((uint64_t)state); - } - - if (state->m_bestIx && state->m_bestIx != &state->m_bestIxBuf) - pfree(state->m_bestIx); - - if (state->m_remoteCondsOrig != nullptr) - list_free(state->m_remoteCondsOrig); - - if (state->m_attrsUsed != NULL) - pfree(state->m_attrsUsed); - - if (state->m_attrsModified != NULL) - pfree(state->m_attrsModified); - - state->m_table = NULL; - pfree(state); -} diff --git a/src/gausskernel/storage/mot/jit_exec/jit_context.cpp b/src/gausskernel/storage/mot/jit_exec/jit_context.cpp index da0a868d8..a0784841b 100644 --- a/src/gausskernel/storage/mot/jit_exec/jit_context.cpp +++ b/src/gausskernel/storage/mot/jit_exec/jit_context.cpp @@ -32,28 +32,74 @@ #include "postgres.h" #include "knl/knl_session.h" #include "storage/ipc.h" +#include "executor/executor.h" + #include "global.h" #include "jit_context.h" #include "jit_context_pool.h" #include "jit_common.h" -#include "jit_tvm.h" #include "mot_internal.h" #include "jit_source.h" #include "mm_global_api.h" +#include "mot_atomic_ops.h" +#include "jit_plan_sp.h" +#include "debug_utils.h" +#include "jit_source_map.h" +#include "jit_statistics.h" namespace JitExec { -DECLARE_LOGGER(JitContext, JitExec); +DECLARE_LOGGER(MotJitContext, JitExec); // The global JIT context pool static JitContextPool g_globalJitCtxPool __attribute__((aligned(64))) = {0}; // forward declarations static JitContextPool* AllocSessionJitContextPool(); -static MOT::Key* PrepareJitSearchKey(JitContext* jitContext, MOT::Index* index); -static void CleanupJitContextPrimary(JitContext* jitContext); -static void CleanupJitContextInner(JitContext* jitContext); -static void CleanupJitContextSubQueryDataArray(JitContext* jitContext); -static void CleanupJitContextSubQueryData(JitContext::SubQueryData* subQueryData); + +static bool AllocJitQueryExecState(JitQueryContext* jitContext); +static bool PrepareJitQueryContext(JitQueryContext* jitContext); +static bool PrepareMainSearchKey(JitQueryContext* jitContext, JitQueryExecState* execState); +static bool PrepareUpdateBitmap(JitQueryContext* jitContext, JitQueryExecState* execState); +static bool PrepareEndIteratorKey(JitQueryContext* jitContext, JitQueryExecState* execState); +static bool PrepareInnerSrearchKey(JitQueryContext* jitContext, JitQueryExecState* execState); +static bool PrepareInnerEndIteratorKey(JitQueryContext* jitContext, JitQueryExecState* execState); +static bool PrepareCompoundSubQuery(JitQueryContext* jitContext, JitQueryExecState* execState); +static bool PrepareInvokeContext(JitQueryContext* jitContext, JitQueryExecState* execState); + +static bool AllocJitFunctionExecState(JitFunctionContext* jitContext); +static bool PrepareJitFunctionContext(JitFunctionContext* jitContext); +static bool PrepareCallSite( + JitFunctionContext* jitContext, JitInvokedQueryExecState* execState, JitCallSite* callSite, int subQueryId); + +static void DestroyJitQueryContext(JitQueryContext* jitContext, bool isDropCachedPlan = false); +static void DestroyJitFunctionContext(JitFunctionContext* jitContext, bool isDropCachedPlan = false); +static MOT::Key* PrepareJitSearchKey(MotJitContext* jitContext, MOT::Index* index); +static bool CloneTupleDesc(TupleDesc source, TupleDesc* target, JitContextUsage usage); +static bool CloneJitQueryContext(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage); +static bool CloneJitFunctionContext(JitFunctionContext* source, JitFunctionContext* target, JitContextUsage usage); +static void CleanupJitContextPrimary(JitQueryContext* jitContext, bool isDropCachedPlan = false); +static void CleanupJitContextInner(JitQueryContext* jitContext, bool isDropCachedPlan = false); +static void CleanupJitSubQueryContextArray(JitQueryContext* jitContext, bool isDropCachedPlan = false); +static void CleanupJitSubQueryContext( + JitSubQueryContext* subQueryContext, JitSubQueryExecState* subQueryExecState, bool isDropCachedPlan = false); +static bool JitQueryContextRefersRelation(JitQueryContext* jitContext, uint64_t relationId); +static bool JitFunctionContextRefersRelation(JitFunctionContext* jitContext, uint64_t relationId); +static void PurgeJitQueryContext(JitQueryContext* jitContext, uint64_t relationId); +static void PurgeJitFunctionContext(JitFunctionContext* jitContext, uint64_t relationId); + +static bool ReplenishDatumArray(JitDatumArray* source, JitDatumArray* target, JitContextUsage usage, int depth); +static bool ReplenishInvokeParams(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage, int depth); +static bool ReplenishSubQueryArray(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage, int depth); +static bool ReplenishAggregateArray(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage, int depth); +static bool ReplenishJitQueryContext(JitQueryContext* source, JitQueryContext* target, int depth); +static bool ReplenishParamListInfo(JitCallSite* target, JitCallSite* source, JitContextUsage usage, int depth); +static bool ReplenishJitFunctionContext(JitFunctionContext* source, JitFunctionContext* target, int depth); +static bool ReplenishJitContext(MotJitContext* source, MotJitContext* target, int depth); +static bool RevalidateJitQueryContext(MotJitContext* jitContext); +static bool RevalidateJitFunctionContext(MotJitContext* jitContext); +static void DestroyJitNonNativeSortExecState( + JitNonNativeSortExecState* jitNonNativeSortExecState, JitContextUsage usage); +static void PropagateValidState(MotJitContext* jitContext, uint8_t invalidFlag, uint64_t relationId); extern bool InitGlobalJitContextPool() { @@ -65,12 +111,15 @@ extern void DestroyGlobalJitContextPool() DestroyJitContextPool(&g_globalJitCtxPool); } -extern JitContext* AllocJitContext(JitContextUsage usage) +extern MotJitContext* AllocJitContext(JitContextUsage usage, JitContextType type) { - JitContext* result = nullptr; - if (usage == JIT_CONTEXT_GLOBAL) { + MotJitContext* result = nullptr; + if (IsJitContextUsageGlobal(usage)) { // allocate from global pool result = AllocPooledJitContext(&g_globalJitCtxPool); + if (result != nullptr) { + JitStatisticsProvider::GetInstance().AddGlobalBytes((int64_t)GetJitContextSize()); + } } else { // allocate from session local pool (create pool on demand and schedule cleanup during end of session) if (u_sess->mot_cxt.jit_session_context_pool == nullptr) { @@ -80,19 +129,47 @@ extern JitContext* AllocJitContext(JitContextUsage usage) } } result = AllocPooledJitContext(u_sess->mot_cxt.jit_session_context_pool); + if (result != nullptr) { + result->m_sessionId = MOT_GET_CURRENT_SESSION_ID(); + JitStatisticsProvider::GetInstance().AddSessionBytes((int64_t)GetJitContextSize()); + ++u_sess->mot_cxt.jit_context_count; + MOT_LOG_TRACE("AllocJitContext(): Current session (%u) JIT context count: %u", + MOT_GET_CURRENT_SESSION_ID(), + u_sess->mot_cxt.jit_context_count); + } + } + + if (result != nullptr) { + result->m_usage = usage; + result->m_contextType = type; + MOT_LOG_TRACE("Allocated %s JIT %s context %p [%u:%u:%u]", + JitContextUsageToString(result->m_usage), + JitContextTypeToString(result->m_contextType), + result, + result->m_poolId, + result->m_subPoolId, + result->m_contextId); + } else { + MOT_LOG_ERROR( + "Failed to allocate %s JIT %s context", JitContextUsageToString(usage), JitContextTypeToString(type)); } return result; } -extern void FreeJitContext(JitContext* jitContext) +extern void FreeJitContext(MotJitContext* jitContext) { if (jitContext != nullptr) { - if (jitContext->m_usage == JIT_CONTEXT_GLOBAL) { + if (IsJitContextUsageGlobal(jitContext->m_usage)) { FreePooledJitContext(&g_globalJitCtxPool, jitContext); + JitStatisticsProvider::GetInstance().AddGlobalBytes(-((int64_t)GetJitContextSize())); } else { // in this scenario it is always called by the session who created the context --u_sess->mot_cxt.jit_context_count; + MOT_LOG_TRACE("FreeJitContext(): Current session (%u) JIT context count: %u", + MOT_GET_CURRENT_SESSION_ID(), + u_sess->mot_cxt.jit_context_count); FreePooledJitContext(u_sess->mot_cxt.jit_session_context_pool, jitContext); + JitStatisticsProvider::GetInstance().AddSessionBytes(-((int64_t)GetJitContextSize())); } } } @@ -107,12 +184,7 @@ static bool CloneDatumArray(JitDatumArray* source, JitDatumArray* target, JitCon } size_t allocSize = sizeof(JitDatum) * datumCount; - JitDatum* datumArray = nullptr; - if (usage == JIT_CONTEXT_GLOBAL) { - datumArray = (JitDatum*)MOT::MemGlobalAlloc(allocSize); - } else { - datumArray = (JitDatum*)MOT::MemSessionAlloc(allocSize); - } + JitDatum* datumArray = (JitDatum*)JitMemAlloc(allocSize, usage); if (datumArray == nullptr) { MOT_REPORT_ERROR( MOT_ERROR_OOM, "JIT Compile", "Failed to allocate %u bytes for datum array", (unsigned)allocSize); @@ -131,18 +203,10 @@ static bool CloneDatumArray(JitDatumArray* source, JitDatumArray* target, JitCon MOT_REPORT_ERROR(MOT_ERROR_OOM, "JIT Compile", "Failed to clone datum array entry"); for (uint32_t j = 0; j < i; ++j) { if (!IsPrimitiveType(datumArray[j].m_type)) { - if (usage == JIT_CONTEXT_GLOBAL) { - MOT::MemGlobalFree(DatumGetPointer(datumArray[j].m_datum)); - } else { - MOT::MemGlobalFree(DatumGetPointer(datumArray[j].m_datum)); - } + JitMemFree(DatumGetPointer(datumArray[j].m_datum), usage); } } - if (usage == JIT_CONTEXT_GLOBAL) { - MOT::MemGlobalFree(datumArray); - } else { - MOT::MemSessionFree(datumArray); - } + JitMemFree(datumArray, usage); return false; } } @@ -154,66 +218,490 @@ static bool CloneDatumArray(JitDatumArray* source, JitDatumArray* target, JitCon return true; } -extern JitContext* CloneJitContext(JitContext* sourceJitContext) +extern MotJitContext* CloneJitContext(MotJitContext* source, JitContextUsage usage) { - MOT_LOG_TRACE("Cloning JIT context %p of query: %s", sourceJitContext, sourceJitContext->m_queryString); - JitContext* result = AllocJitContext(JIT_CONTEXT_LOCAL); // clone is always for local use - if (result == nullptr) { + const char* itemName = (source->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) ? "query" : "function"; + MOT_LOG_TRACE("Cloning %s JIT context %p (valid state %x) of %s: %s", + JitContextUsageToString(source->m_usage), + source, + MOT_ATOMIC_LOAD(source->m_validState), + itemName, + source->m_queryString); + MotJitContext* target = AllocJitContext(usage, source->m_contextType); + if (target == nullptr) { MOT_LOG_TRACE("Failed to allocate JIT context object"); return nullptr; } - result->m_llvmFunction = sourceJitContext->m_llvmFunction; - result->m_tvmFunction = sourceJitContext->m_tvmFunction; - result->m_commandType = sourceJitContext->m_commandType; - if (!CloneDatumArray(&sourceJitContext->m_constDatums, &result->m_constDatums, JIT_CONTEXT_LOCAL)) { + // no need to clone module/code-gen object (they are safe in the source context) + target->m_llvmFunction = source->m_llvmFunction; + target->m_llvmSPFunction = source->m_llvmSPFunction; + target->m_commandType = source->m_commandType; + target->m_validState = source->m_validState; + target->m_queryString = source->m_queryString; + AddJitSourceContext(source->m_jitSource, target); // register target for cleanup due to DDL + if (!CloneDatumArray(&source->m_constDatums, &target->m_constDatums, usage)) { MOT_REPORT_ERROR(MOT_ERROR_OOM, "JIT Compile", "Failed to clone constant datum array"); - DestroyJitContext(result); + DestroyJitContext(target); return nullptr; } - result->m_table = sourceJitContext->m_table; - result->m_index = sourceJitContext->m_index; - result->m_indexId = sourceJitContext->m_indexId; - result->m_argCount = sourceJitContext->m_argCount; - result->m_nullColumnId = sourceJitContext->m_nullColumnId; - result->m_queryString = sourceJitContext->m_queryString; - result->m_innerTable = sourceJitContext->m_innerTable; - result->m_innerIndex = sourceJitContext->m_innerIndex; - result->m_innerIndexId = sourceJitContext->m_innerIndexId; - result->m_subQueryCount = sourceJitContext->m_subQueryCount; + + bool result = true; + if (source->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + result = CloneJitQueryContext((JitQueryContext*)source, (JitQueryContext*)target, usage); + } else if (source->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + result = CloneJitFunctionContext((JitFunctionContext*)source, (JitFunctionContext*)target, usage); + } + + if (!result) { + MOT_LOG_TRACE("Failed to clone %s JIT context %p: %s", + JitContextUsageToString(source->m_usage), + source, + itemName, + source->m_queryString); + DestroyJitContext(target); + target = nullptr; + } else { + if (source->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + MOT_LOG_TRACE("Cloned %s query JIT context %p into %s JIT context %p (table=%p): %s", + JitContextUsageToString(source->m_usage), + source, + JitContextUsageToString(usage), + target, + ((JitQueryContext*)target)->m_table, + source->m_queryString); + } else { + MOT_LOG_TRACE("Cloned %s function JIT context %p into %s JIT context %p (procid=%u): %s", + JitContextUsageToString(source->m_usage), + source, + JitContextUsageToString(usage), + target, + ((JitFunctionContext*)target)->m_functionOid, + source->m_queryString); + } + } + + return target; +} + +static void DestroyJitNonNativeSortParamsMembers(JitNonNativeSortParams* sortParams, JitContextUsage usage) +{ + if (sortParams->sortColIdx) { + JitMemFree(sortParams->sortColIdx, usage); + sortParams->sortColIdx = nullptr; + } + if (sortParams->sortOperators) { + JitMemFree(sortParams->sortOperators, usage); + sortParams->sortOperators = nullptr; + } + if (sortParams->collations) { + JitMemFree(sortParams->collations, usage); + sortParams->collations = nullptr; + } + if (sortParams->nullsFirst) { + JitMemFree(sortParams->nullsFirst, usage); + sortParams->nullsFirst = nullptr; + } +} + +extern void DestroyJitNonNativeSortParams(JitNonNativeSortParams* sortParams, JitContextUsage usage) +{ + if (sortParams) { + DestroyJitNonNativeSortParamsMembers(sortParams, usage); + JitMemFree(sortParams, usage); + } +} + +static bool AllocJitNonNativeSortParamsMembers(JitNonNativeSortParams* sortParams, int numCols, JitContextUsage usage) +{ + if (numCols < 1) { + MOT_LOG_ERROR("Cannot create JitNonNativeSortParams if numCols is %d", numCols); + return false; + } + + sortParams->numCols = numCols; + uint32_t totalAllocSize = numCols * sizeof(AttrNumber) + 2 * numCols * sizeof(Oid) + numCols * sizeof(bool); + + sortParams->sortColIdx = (AttrNumber*)JitMemAlloc(numCols * sizeof(AttrNumber), usage); + sortParams->sortOperators = (Oid*)JitMemAlloc(numCols * sizeof(Oid), usage); + sortParams->collations = (Oid*)JitMemAlloc(numCols * sizeof(Oid), usage); + sortParams->nullsFirst = (bool*)JitMemAlloc(numCols * sizeof(bool), usage); + if (sortParams->sortColIdx == nullptr || sortParams->sortOperators == nullptr || + sortParams->collations == nullptr || sortParams->nullsFirst == nullptr) { + MOT_LOG_TRACE("Generate JIT Code", + "AllocJitNonNativeSortParamsMembers(): Failed to allocate memory for JitNonNativeSortParams members. " + "total alloc size = %u, numCols = d%, sortColIdx = %p, sortOperators = %p, collations = %p, nullsFirst = " + "%p", + totalAllocSize, + numCols, + sortParams->sortColIdx, + sortParams->sortOperators, + sortParams->collations, + sortParams->nullsFirst); + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate memory of size %u for Non-native sort params.", + totalAllocSize); + DestroyJitNonNativeSortParamsMembers(sortParams, usage); + return false; + } + + errno_t erc; + erc = memset_s(sortParams->sortColIdx, numCols * sizeof(AttrNumber), 0, numCols * sizeof(AttrNumber)); + securec_check(erc, "\0", "\0"); + erc = memset_s(sortParams->sortOperators, numCols * sizeof(Oid), 0, numCols * sizeof(Oid)); + securec_check(erc, "\0", "\0"); + erc = memset_s(sortParams->collations, numCols * sizeof(Oid), 0, numCols * sizeof(Oid)); + securec_check(erc, "\0", "\0"); + erc = memset_s(sortParams->nullsFirst, numCols * sizeof(bool), 0, numCols * sizeof(bool)); + securec_check(erc, "\0", "\0"); + + return true; +} + +extern JitNonNativeSortParams* AllocJitNonNativeSortParams(int numCols, JitContextUsage usage) +{ + uint64_t allocSize = sizeof(struct JitNonNativeSortParams); + JitNonNativeSortParams* jitNonNativeSortParams = (JitNonNativeSortParams*)JitMemAlloc(allocSize, usage); + if (jitNonNativeSortParams == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %u bytes for JitNonNativeSortParams", + (unsigned)allocSize); + return nullptr; + } + + errno_t erc = memset_s( + jitNonNativeSortParams, sizeof(struct JitNonNativeSortParams), 0, sizeof(struct JitNonNativeSortParams)); + securec_check(erc, "\0", "\0"); + + if (!AllocJitNonNativeSortParamsMembers(jitNonNativeSortParams, numCols, usage)) { + MOT_LOG_ERROR("Failed to allocate JitNonNativeSortParams members. numcols: %d", numCols); + DestroyJitNonNativeSortParams(jitNonNativeSortParams, usage); + jitNonNativeSortParams = nullptr; + } + + return jitNonNativeSortParams; +} + +// Assume all memory in dest object was already allocated +bool CopyJitNonNativeSortParams(JitNonNativeSortParams* src, JitNonNativeSortParams* dest) +{ + MOT_ASSERT(src->numCols == dest->numCols); + + if (src->numCols != dest->numCols) { + MOT_LOG_ERROR("Cannot copy Non native sort params. src numCols (%d) is not " + "equal to dest numCols (%d)", + src->numCols, + dest->numCols); + + return false; + } + + int numCols = src->numCols; + + // Copy data + errno_t erc; + erc = memcpy_s(dest->sortColIdx, numCols * sizeof(AttrNumber), src->sortColIdx, numCols * sizeof(AttrNumber)); + securec_check(erc, "\0", "\0"); + + erc = memcpy_s(dest->sortOperators, numCols * sizeof(Oid), src->sortOperators, numCols * sizeof(Oid)); + securec_check(erc, "\0", "\0"); + + erc = memcpy_s(dest->collations, numCols * sizeof(Oid), src->collations, numCols * sizeof(Oid)); + securec_check(erc, "\0", "\0"); + + erc = memcpy_s(dest->nullsFirst, numCols * sizeof(bool), src->nullsFirst, numCols * sizeof(bool)); + securec_check(erc, "\0", "\0"); + + dest->numCols = src->numCols; + dest->plan_node_id = src->plan_node_id; + dest->bound = src->bound; + dest->scanDir = src->scanDir; + + return true; +} + +extern JitNonNativeSortParams* CloneJitNonNativeSortParams(JitNonNativeSortParams* src, JitContextUsage usage) +{ + JitNonNativeSortParams* jitNonNativeSortParams = AllocJitNonNativeSortParams(src->numCols, usage); + + if (jitNonNativeSortParams == nullptr) { + MOT_LOG_ERROR("Failed to allocate JitNonNativeSortParams with %d columns", src->numCols); + + return nullptr; + } + + if (!CopyJitNonNativeSortParams(src, jitNonNativeSortParams)) { + MOT_LOG_ERROR("Failed to copy non native params with %d columns", src->numCols); + + DestroyJitNonNativeSortParams(jitNonNativeSortParams, usage); + jitNonNativeSortParams = nullptr; + + return nullptr; + } + + return jitNonNativeSortParams; +} + +static inline void JitQueryContextSetTablesAndIndices(JitQueryContext* source, JitQueryContext* target) +{ + target->m_table = source->m_table; + target->m_tableId = source->m_tableId; + target->m_index = source->m_index; + target->m_indexId = source->m_indexId; + target->m_innerTable = source->m_innerTable; + target->m_innerTableId = source->m_innerTableId; + target->m_innerIndex = source->m_innerIndex; + target->m_innerIndexId = source->m_innerIndexId; +} + +static bool CloneJitQueryContext(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage) +{ + JitQueryContextSetTablesAndIndices(source, target); + target->m_aggCount = source->m_aggCount; + target->m_subQueryCount = source->m_subQueryCount; + target->m_invokeParamCount = source->m_invokeParamCount; + target->m_subQueryContext = nullptr; + target->m_invokeContext = nullptr; + + if (source->m_invokeContext != nullptr) { + MOT_LOG_TRACE("Cloning invoked stored procedure context"); + target->m_invokeContext = (JitFunctionContext*)CloneJitContext(source->m_invokeContext, usage); + if (target->m_invokeContext == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone invocation context"); + return false; + } + target->m_invokeContext->m_parentContext = target; + } + if (source->m_invokeParamCount > 0) { + size_t allocSize = sizeof(JitParamInfo) * source->m_invokeParamCount; + target->m_invokeParamInfo = (JitParamInfo*)JitMemAlloc(allocSize, usage); + if (target->m_invokeParamInfo == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %u bytes for %u invoke parameter mode array items in JIT context object", + (unsigned)allocSize, + (unsigned)source->m_invokeParamCount); + return false; + } + errno_t erc = memcpy_s(target->m_invokeParamInfo, allocSize, source->m_invokeParamInfo, allocSize); + securec_check(erc, "\0", "\0"); + } else { + target->m_invokeParamInfo = nullptr; + } + + MOT_ASSERT((target->m_invokeParamCount == 0 && target->m_invokeParamInfo == nullptr) || + (target->m_invokeParamCount > 0 && target->m_invokeParamInfo != nullptr)); + + if (source->m_invokedQueryString != nullptr) { + target->m_invokedQueryString = DupString(source->m_invokedQueryString, usage); + if (target->m_invokedQueryString == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to duplicate invoke query string: %s", + source->m_invokedQueryString); + return false; + } + } + target->m_invokedFunctionOid = source->m_invokedFunctionOid; + target->m_invokedFunctionTxnId = source->m_invokedFunctionTxnId; // clone sub-query tuple descriptor array - MOT_LOG_TRACE("Cloning %u sub-query data items", (unsigned)sourceJitContext->m_subQueryCount); - if (sourceJitContext->m_subQueryCount > 0) { - uint32_t allocSize = sizeof(JitContext::SubQueryData) * sourceJitContext->m_subQueryCount; - result->m_subQueryData = (JitContext::SubQueryData*)MOT::MemGlobalAllocAligned(allocSize, L1_CACHE_LINE); - if (result->m_subQueryData == nullptr) { + if (target->m_subQueryCount > 0) { + MOT_LOG_TRACE("Cloning %u sub-query data items", (unsigned)source->m_subQueryCount); + size_t allocSize = sizeof(JitSubQueryContext) * source->m_subQueryCount; + target->m_subQueryContext = (JitSubQueryContext*)JitMemAllocAligned(allocSize, L1_CACHE_LINE, usage); + if (target->m_subQueryContext == nullptr) { MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to allocate %u bytes for %u sub-query data array in JIT context object", - allocSize, - (unsigned)sourceJitContext->m_subQueryCount); - FreeJitContext(result); - return nullptr; + (unsigned)allocSize, + (unsigned)source->m_subQueryCount); + return false; } - for (uint32_t i = 0; i < sourceJitContext->m_subQueryCount; ++i) { - // copy known members - result->m_subQueryData[i].m_commandType = sourceJitContext->m_subQueryData[i].m_commandType; - result->m_subQueryData[i].m_table = sourceJitContext->m_subQueryData[i].m_table; - result->m_subQueryData[i].m_index = sourceJitContext->m_subQueryData[i].m_index; - result->m_subQueryData[i].m_indexId = sourceJitContext->m_subQueryData[i].m_indexId; - - // nullify other members - result->m_subQueryData[i].m_tupleDesc = nullptr; - result->m_subQueryData[i].m_slot = nullptr; - result->m_subQueryData[i].m_searchKey = nullptr; - result->m_subQueryData[i].m_endIteratorKey = nullptr; + for (uint32_t i = 0; i < source->m_subQueryCount; ++i) { + target->m_subQueryContext[i].m_commandType = source->m_subQueryContext[i].m_commandType; + target->m_subQueryContext[i].m_table = source->m_subQueryContext[i].m_table; + target->m_subQueryContext[i].m_tableId = source->m_subQueryContext[i].m_tableId; + target->m_subQueryContext[i].m_index = source->m_subQueryContext[i].m_index; + target->m_subQueryContext[i].m_indexId = source->m_subQueryContext[i].m_indexId; } } - MOT_LOG_TRACE("Cloned JIT context %p into %p (table=%p)", sourceJitContext, result, result->m_table); - return result; + if (source->m_nonNativeSortParams) { + target->m_nonNativeSortParams = CloneJitNonNativeSortParams(source->m_nonNativeSortParams, usage); + if (target->m_nonNativeSortParams == nullptr) { + MOT_LOG_ERROR("Failed to clone JitNonNativeSortParams object with %d columns", + source->m_nonNativeSortParams->numCols); + return false; + } + } + + MOT_LOG_TRACE("Cloned JIT context %p into %p (table=%p)", source, target, target->m_table); + return true; +} + +static bool CloneCallSite( + JitFunctionContext* jitContext, int subQueryId, JitCallSite* source, JitCallSite* target, JitContextUsage usage) +{ + if (source->m_queryContext == nullptr) { + target->m_queryContext = nullptr; + target->m_queryString = DupString(source->m_queryString, usage); + if (target->m_queryString == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to clone call site query string: %s", + source->m_queryString); + return false; + } + MOT_LOG_TRACE("CloneCallSite(): Cloned string %p on %s scope into call site %p: %s", + target->m_queryString, + IsJitContextUsageGlobal(usage) ? "global" : "local", + target, + target->m_queryString); + } else { + target->m_queryContext = CloneJitContext(source->m_queryContext, usage); + if (target->m_queryContext == nullptr) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone sub-query context for function JIT context"); + return false; + } + target->m_queryString = nullptr; + } + target->m_queryCmdType = source->m_queryCmdType; + + target->m_callParamCount = source->m_callParamCount; + if (target->m_callParamCount > 0) { + size_t allocSize = sizeof(JitCallParamInfo) * target->m_callParamCount; + target->m_callParamInfo = (JitCallParamInfo*)JitMemAlloc(allocSize, usage); + if (target->m_callParamInfo == nullptr) { + MOT_LOG_ERROR("Failed to allocate %u bytes for %d call parameters", allocSize, target->m_callParamCount); + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone sub-query parameters for function JIT context"); + return false; + } + errno_t erc = memcpy_s(target->m_callParamInfo, allocSize, source->m_callParamInfo, allocSize); + securec_check(erc, "\0", "\0"); + } else { + target->m_callParamInfo = nullptr; + } + + if (!CloneTupleDesc(source->m_tupDesc, &target->m_tupDesc, usage)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone sub-query tuple descriptor"); + return false; + } + target->m_exprIndex = source->m_exprIndex; + target->m_isUnjittableInvoke = source->m_isUnjittableInvoke; + target->m_isModStmt = source->m_isModStmt; + target->m_isInto = source->m_isInto; + return true; +} + +static bool CloneTupleDesc(TupleDesc source, TupleDesc* target, JitContextUsage usage) +{ + if (source != nullptr) { + MemoryContext oldCtx = CurrentMemoryContext; + if (IsJitContextUsageGlobal(usage)) { + CurrentMemoryContext = INSTANCE_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR); + } else { + CurrentMemoryContext = SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR); + } + *target = CreateTupleDescCopy(source); + CurrentMemoryContext = oldCtx; + if (*target == nullptr) { + return false; + } + } else { + *target = nullptr; + } + return true; +} + +static bool CloneResultDescriptors(JitFunctionContext* source, JitFunctionContext* target) +{ + JitContextUsage usage = target->m_usage; + + target->m_compositeResult = source->m_compositeResult; + + if (!CloneTupleDesc(source->m_resultTupDesc, &target->m_resultTupDesc, usage)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone result tuple descriptor for function JIT context"); + return false; + } + + if (!CloneTupleDesc(source->m_rowTupDesc, &target->m_rowTupDesc, usage)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone row tuple descriptor for function JIT context"); + return false; + } + + return true; +} + +static bool CloneJitFunctionContext(JitFunctionContext* source, JitFunctionContext* target, JitContextUsage usage) +{ + target->m_functionOid = source->m_functionOid; + target->m_functionTxnId = source->m_functionTxnId; + target->m_SPArgCount = source->m_SPArgCount; + target->m_SPSubQueryCount = source->m_SPSubQueryCount; + if (target->m_SPSubQueryCount == 0) { + target->m_SPSubQueryList = nullptr; + return true; + } + + // in any case of failure caller will call destroy function for safe cleanup + size_t allocSize = sizeof(JitCallSite) * target->m_SPSubQueryCount; + target->m_SPSubQueryList = (JitCallSite*)JitMemAlloc(allocSize, usage); + if (target->m_SPSubQueryList == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %u bytes for %u sub-query data array in Function JIT context object", + allocSize, + (unsigned)target->m_SPSubQueryCount); + return false; + } + errno_t erc = memset_s(target->m_SPSubQueryList, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + + for (uint32_t i = 0; i < target->m_SPSubQueryCount; ++i) { + JitCallSite* sourceCallSite = &source->m_SPSubQueryList[i]; + JitCallSite* targetCallSite = &target->m_SPSubQueryList[i]; + if (!CloneCallSite(target, i, sourceCallSite, targetCallSite, usage)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone call-site %d for function JIT context", i); + return false; + } + if (targetCallSite->m_queryContext != nullptr) { + targetCallSite->m_queryContext->m_parentContext = target; + } + } + target->m_paramCount = source->m_paramCount; + if (target->m_paramCount == 0) { + target->m_paramTypes = nullptr; + } else { + allocSize = sizeof(Oid) * target->m_paramCount; + target->m_paramTypes = (Oid*)JitMemAlloc(allocSize, usage); + } + if (target->m_paramTypes == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %u bytes for %u sub-query parameter array in Function JIT context object", + allocSize, + target->m_paramCount); + return false; + } + erc = memcpy_s(target->m_paramTypes, allocSize, source->m_paramTypes, allocSize); + securec_check(erc, "\0", "\0"); + + if (!CloneResultDescriptors(source, target)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone result descriptors for function JIT context"); + return false; + } + + return true; } static inline List* GetSubExprTargetList(Query* query, Expr* expr, int subQueryIndex, int* subQueryCount); @@ -301,47 +789,140 @@ static List* GetSubQueryTargetList(const char* queryString, int subQueryIndex) return targetList; } -extern bool ReFetchIndices(JitContext* jitContext) +static bool RefetchJitFunctionContext(JitFunctionContext* functionContext) { - if (jitContext->m_commandType == JIT_COMMAND_INSERT) { + MOT_LOG_TRACE("Re-fetching tables and indices for function context %p with query: %s", + functionContext, + functionContext->m_queryString); + for (uint32_t i = 0; i < functionContext->m_SPSubQueryCount; ++i) { + JitCallSite* callSite = &functionContext->m_SPSubQueryList[i]; + if (callSite->m_queryContext != nullptr) { + MOT_LOG_TRACE("Re-fetching tables and indices for sub-query %u in function context %p with query: %s", + i, + functionContext, + callSite->m_queryContext->m_queryString); + if (!RefetchTablesAndIndices(callSite->m_queryContext)) { + return false; + } + } + } + return true; +} + +extern bool RefetchTablesAndIndices(MotJitContext* jitContext) +{ + MOT_LOG_TRACE( + "Re-fetching tables and indices for context %p with query: %s", jitContext, jitContext->m_queryString); + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + // handle function context + return RefetchJitFunctionContext((JitFunctionContext*)jitContext); + } + + // handle invoke context + JitQueryContext* queryContext = (JitQueryContext*)jitContext; + if (jitContext->m_commandType == JIT_COMMAND_INVOKE) { + MOT_LOG_TRACE("Re-fetching tables and indices for INVOKE context %p with query: %s", + jitContext, + jitContext->m_queryString); + if (queryContext->m_invokeContext != nullptr) { + return RefetchTablesAndIndices(queryContext->m_invokeContext); + } return true; } - // re-fetch main index - if (jitContext->m_index == nullptr) { - jitContext->m_index = jitContext->m_table->GetIndexByExtId(jitContext->m_indexId); - if (jitContext->m_index == nullptr) { - MOT_LOG_TRACE("Failed to fetch index by extern id %" PRIu64 " for query: %s", - jitContext->m_indexId, - jitContext->m_queryString); + // re-fetch main table + MOT::TxnManager* currTxn = GetSafeTxn(__FUNCTION__); + MOT_ASSERT(currTxn != nullptr); + if (queryContext->m_table == nullptr) { + MOT_LOG_TRACE( + "Re-fetching main table in query context %p with query: %s", jitContext, jitContext->m_queryString); + queryContext->m_table = currTxn->GetTableByExternalId(queryContext->m_tableId); + if (queryContext->m_table == nullptr) { + MOT_LOG_TRACE("Failed to fetch table by extern id %" PRIu64 " for query: %s", + queryContext->m_tableId, + queryContext->m_queryString); return false; } - MOT_LOG_TRACE("Fetched index %s by extern id %" PRIu64 " for query: %s", - jitContext->m_index->GetName().c_str(), - jitContext->m_indexId, - jitContext->m_queryString); + MOT_LOG_TRACE("Fetched table %s (%p) by extern id %" PRIu64 " for query: %s", + queryContext->m_table->GetTableName().c_str(), + queryContext->m_table, + queryContext->m_tableId, + queryContext->m_queryString); } - // re-fetch inner index (JOIN commands only) - if (IsJoinCommand(jitContext->m_commandType)) { - if (jitContext->m_innerIndex == nullptr) { - jitContext->m_innerIndex = jitContext->m_innerTable->GetIndexByExtId(jitContext->m_innerIndexId); - if (jitContext->m_innerIndex == nullptr) { - MOT_LOG_TRACE("Failed to fetch inner index by extern id %" PRIu64, jitContext->m_innerIndexId); + // re-fetch main index + if (queryContext->m_index == nullptr) { + MOT_LOG_TRACE("Re-fetching indices for main table in query context %p with query: %s", + jitContext, + jitContext->m_queryString); + queryContext->m_index = queryContext->m_table->GetIndexByExtId(queryContext->m_indexId); + if (queryContext->m_index == nullptr) { + MOT_LOG_TRACE("Failed to fetch index by extern id %" PRIu64 " for query: %s", + queryContext->m_indexId, + queryContext->m_queryString); + return false; + } + MOT_LOG_TRACE("Fetched index %s (%p) by extern id %" PRIu64 " for query: %s", + queryContext->m_index->GetName().c_str(), + queryContext->m_index, + queryContext->m_indexId, + queryContext->m_queryString); + } + + // re-fetch inner table and index (JOIN commands only) + if (IsJoinCommand(queryContext->m_commandType)) { + if (queryContext->m_innerTable == nullptr) { + MOT_LOG_TRACE("Re-fetching inner table in JOIN query context %p with query: %s", + jitContext, + jitContext->m_queryString); + queryContext->m_innerTable = currTxn->GetTableByExternalId(queryContext->m_innerTableId); + if (queryContext->m_innerTable == nullptr) { + MOT_LOG_TRACE("Failed to fetch inner table by extern id %" PRIu64, queryContext->m_innerTableId); + return false; + } + } + + if (queryContext->m_innerIndex == nullptr) { + MOT_LOG_TRACE("Re-fetching indices for inner table in JOIN query context %p with query: %s", + jitContext, + jitContext->m_queryString); + queryContext->m_innerIndex = queryContext->m_innerTable->GetIndexByExtId(queryContext->m_innerIndexId); + if (queryContext->m_innerIndex == nullptr) { + MOT_LOG_TRACE("Failed to fetch inner index by extern id %" PRIu64, queryContext->m_innerIndexId); return false; } } } - // re-fetch sub-query indices (COMPOUND commands only) - if (jitContext->m_commandType == JIT_COMMAND_COMPOUND_SELECT) { - for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { - JitContext::SubQueryData* subQueryData = &jitContext->m_subQueryData[i]; - if (subQueryData->m_index == nullptr) { - subQueryData->m_index = subQueryData->m_table->GetIndexByExtId(subQueryData->m_indexId); - if (subQueryData->m_index == nullptr) { + // re-fetch sub-query tables and indices (COMPOUND commands only) + if (queryContext->m_commandType == JIT_COMMAND_COMPOUND_SELECT) { + MOT_LOG_TRACE("Re-fetching tables and indices for compound query context %p with query: %s", + jitContext, + jitContext->m_queryString); + for (uint32_t i = 0; i < queryContext->m_subQueryCount; ++i) { + JitSubQueryContext* subQueryContext = &queryContext->m_subQueryContext[i]; + if (subQueryContext->m_table == nullptr) { + MOT_LOG_TRACE("Re-fetching table for compound sub-query %u in context %p with query: %s", + i, + jitContext, + jitContext->m_queryString); + subQueryContext->m_table = currTxn->GetTableByExternalId(subQueryContext->m_tableId); + if (subQueryContext->m_table == nullptr) { MOT_LOG_TRACE( - "Failed to fetch sub-query %u index by extern id %" PRIu64, i, subQueryData->m_indexId); + "Failed to fetch sub-query %u table by extern id %" PRIu64, i, subQueryContext->m_tableId); + return false; + } + } + + if (subQueryContext->m_index == nullptr) { + MOT_LOG_TRACE("Re-fetching indices for compound sub-query %u in context %p with query: %s", + i, + jitContext, + jitContext->m_queryString); + subQueryContext->m_index = subQueryContext->m_table->GetIndexByExtId(subQueryContext->m_indexId); + if (subQueryContext->m_index == nullptr) { + MOT_LOG_TRACE( + "Failed to fetch sub-query %u index by extern id %" PRIu64, i, subQueryContext->m_indexId); return false; } } @@ -351,205 +932,848 @@ extern bool ReFetchIndices(JitContext* jitContext) return true; } -static bool PrepareJitContextJoinData(JitContext* jitContext) +extern bool PrepareJitContext(MotJitContext* jitContext) { - // allocate inner loop search key for JOIN commands - if ((jitContext->m_innerSearchKey == nullptr) && IsJoinCommand(jitContext->m_commandType)) { - MOT_LOG_TRACE( - "Preparing inner search key for JOIN command from index %s", jitContext->m_innerIndex->GetName().c_str()); - jitContext->m_innerSearchKey = PrepareJitSearchKey(jitContext, jitContext->m_innerIndex); - if (jitContext->m_innerSearchKey == nullptr) { - MOT_LOG_TRACE( - "Failed to allocate reusable inner search key for JIT context, aborting jitted code execution"); - return false; // safe cleanup during destroy + bool result = false; + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + result = PrepareJitQueryContext((JitQueryContext*)jitContext); + } else { + result = PrepareJitFunctionContext((JitFunctionContext*)jitContext); + } + // reset purged flag on success, otherwise make sure it is still raised, so that next execution round will attempt + // again to prepare the context before execution + if (jitContext->m_execState != nullptr) { + if (result) { + MOT_ATOMIC_STORE(jitContext->m_execState->m_purged, false); + } else { + MOT_ATOMIC_STORE(jitContext->m_execState->m_purged, true); + } + } + return result; +} + +static bool AllocJitQueryExecState(JitQueryContext* jitContext) +{ + size_t allocSize = sizeof(JitQueryExecState); + JitQueryExecState* execState = (JitQueryExecState*)MOT::MemSessionAlloc(allocSize); + if (execState == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT Query", + "Failed to allocate %u bytes for query execution state", + (unsigned)allocSize); + return false; + } + + errno_t erc = memset_s(execState, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + + // allocate aggregate array + if (jitContext->m_aggCount > 0) { + allocSize = sizeof(JitAggExecState) * jitContext->m_aggCount; + execState->m_aggExecState = (JitAggExecState*)MOT::MemSessionAlloc(allocSize); + if (execState->m_aggExecState == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT Query", + "Failed to allocate %u bytes for query aggregate execution state", + (unsigned)allocSize); + MOT::MemSessionFree(execState); + return false; } - MOT_LOG_TRACE("Prepared inner search key %p (%u bytes) for JOIN command from index %s", - jitContext->m_innerSearchKey, - jitContext->m_innerSearchKey->GetKeyLength(), - jitContext->m_innerIndex->GetName().c_str()); + errno_t erc = memset_s(execState->m_aggExecState, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + } + + // allocate sub-query data array for compound commands + if (jitContext->m_commandType == JIT_COMMAND_COMPOUND_SELECT) { + execState->m_subQueryCount = jitContext->m_subQueryCount; + allocSize = sizeof(JitSubQueryExecState) * execState->m_subQueryCount; + execState->m_subQueryExecState = (JitSubQueryExecState*)MOT::MemSessionAlloc(allocSize); + if (execState->m_subQueryExecState == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT Query", + "Failed to allocate %u bytes for sub-query data in query execution state", + (unsigned)allocSize); + MOT::MemSessionFree(execState->m_aggExecState); + execState->m_aggExecState = nullptr; + MOT::MemSessionFree(execState); + execState = nullptr; + return false; + } + + errno_t erc = memset_s(execState->m_subQueryExecState, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + } + + if (jitContext->m_nonNativeSortParams) { + MOT_ASSERT(jitContext->m_commandType == JIT_COMMAND_RANGE_SELECT); + allocSize = sizeof(JitNonNativeSortExecState); + execState->m_nonNativeSortExecState = (JitNonNativeSortExecState*)MOT::MemSessionAlloc(allocSize); + + if (execState->m_nonNativeSortExecState == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT Query", + "Failed to allocate %u bytes for JitNonNativeSortExecState in query execution state", + (unsigned)allocSize); + + if (execState->m_subQueryExecState) { + MOT::MemSessionFree(execState->m_subQueryExecState); + execState->m_subQueryExecState = nullptr; + } + + if (execState->m_aggExecState) { + MOT::MemSessionFree(execState->m_aggExecState); + execState->m_aggExecState = nullptr; + } + MOT::MemSessionFree(execState); + execState = nullptr; + + return false; + } + + execState->m_nonNativeSortExecState->m_tupleSort = nullptr; + } + + jitContext->m_execState = (JitExecState*)execState; + return true; +} + +static bool PrepareMainSearchKey(JitQueryContext* jitContext, JitQueryExecState* execState) +{ + MOT_LOG_TRACE("Preparing search key from index %s", jitContext->m_index->GetName().c_str()); + execState->m_searchKey = PrepareJitSearchKey(jitContext, jitContext->m_index); + if (execState->m_searchKey == nullptr) { + MOT_LOG_TRACE("Failed to allocate reusable search key for JIT context, aborting jitted code execution"); + return false; + } + + MOT_LOG_TRACE("Prepared search key %p (%u bytes) from index %s", + execState->m_searchKey, + execState->m_searchKey->GetKeyLength(), + jitContext->m_index->GetName().c_str()); + return true; +} + +static bool PrepareUpdateBitmap(JitQueryContext* jitContext, JitQueryExecState* execState) +{ + int fieldCount = (int)jitContext->m_table->GetFieldCount(); + MOT_LOG_TRACE("Initializing reusable bitmap set according to %d fields (including null-bits column 0) in table %s", + fieldCount, + jitContext->m_table->GetLongTableName().c_str()); + void* buf = MOT::MemSessionAlloc(sizeof(MOT::BitmapSet)); + if (buf == nullptr) { + MOT_LOG_TRACE("Failed to allocate reusable bitmap set for JIT context, aborting jitted code execution"); + return false; // safe cleanup during destroy + } + + uint8_t* bitmapData = (uint8_t*)MOT::MemSessionAlloc(MOT::BitmapSet::GetLength(fieldCount)); + if (bitmapData == nullptr) { + MOT_LOG_TRACE("Failed to allocate reusable bitmap set for JIT context, aborting jitted code execution"); + MOT::MemSessionFree(buf); + return false; // safe cleanup during destroy + } + execState->m_bitmapSet = new (buf) MOT::BitmapSet(bitmapData, fieldCount); + return true; +} + +static bool PrepareEndIteratorKey(JitQueryContext* jitContext, JitQueryExecState* execState) +{ + MOT_LOG_TRACE("Preparing end iterator key for range update/select command from index %s", + jitContext->m_index->GetName().c_str()); + execState->m_endIteratorKey = PrepareJitSearchKey(jitContext, jitContext->m_index); + if (execState->m_endIteratorKey == nullptr) { + MOT_LOG_TRACE("Failed to allocate reusable end iterator key for JIT context, aborting jitted code execution"); + return false; // safe cleanup during destroy + } + + MOT_LOG_TRACE("Prepared end iterator key %p (%u bytes) for range update/select command from index %s", + execState->m_endIteratorKey, + execState->m_endIteratorKey->GetKeyLength(), + jitContext->m_index->GetName().c_str()); + return true; +} + +static bool PrepareInnerSrearchKey(JitQueryContext* jitContext, JitQueryExecState* execState) +{ + MOT_LOG_TRACE( + "Preparing inner search key for JOIN command from index %s", jitContext->m_innerIndex->GetName().c_str()); + execState->m_innerSearchKey = PrepareJitSearchKey(jitContext, jitContext->m_innerIndex); + if (execState->m_innerSearchKey == nullptr) { + MOT_LOG_TRACE("Failed to allocate reusable inner search key for JIT context, aborting jitted code execution"); + return false; // safe cleanup during destroy + } + + MOT_LOG_TRACE("Prepared inner search key %p (%u bytes) for JOIN command from index %s", + execState->m_innerSearchKey, + execState->m_innerSearchKey->GetKeyLength(), + jitContext->m_innerIndex->GetName().c_str()); + return true; +} + +static bool PrepareInnerEndIteratorKey(JitQueryContext* jitContext, JitQueryExecState* execState) +{ + MOT_LOG_TRACE( + "Preparing inner end iterator key for JOIN command from index %s", jitContext->m_innerIndex->GetName().c_str()); + execState->m_innerEndIteratorKey = PrepareJitSearchKey(jitContext, jitContext->m_innerIndex); + if (execState->m_innerEndIteratorKey == nullptr) { + MOT_LOG_TRACE( + "Failed to allocate reusable inner end iterator key for JIT context, aborting jitted code execution"); + return false; // safe cleanup during destroy + } + + MOT_LOG_TRACE("Prepared inner end iterator key %p (%u bytes) for JOIN command from index %s", + execState->m_innerEndIteratorKey, + execState->m_innerEndIteratorKey->GetKeyLength(), + jitContext->m_innerIndex->GetName().c_str()); + return true; +} + +static bool PrepareCompoundSubQuery(JitQueryContext* jitContext, JitQueryExecState* execState) +{ + // allocate sub-query search keys and generate tuple table slot array using session top memory context + for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { + JitSubQueryContext* subQueryContext = &jitContext->m_subQueryContext[i]; + JitSubQueryExecState* subQueryExecState = &execState->m_subQueryExecState[i]; + if (subQueryExecState->m_tupleDesc == nullptr) { + MOT_LOG_TRACE("Preparing sub-query %u tuple descriptor", i); + List* targetList = GetSubQueryTargetList(jitContext->m_queryString, i); + if (targetList == nullptr) { + MOT_LOG_TRACE("Failed to locate sub-query %u target list", i); + return false; // safe cleanup during destroy + } else { + subQueryExecState->m_tupleDesc = ExecCleanTypeFromTL(targetList, false); + if (subQueryExecState->m_tupleDesc == nullptr) { + MOT_LOG_TRACE("Failed to create sub-query %u tuple descriptor from target list", i); + return false; // safe cleanup during destroy + } + Assert(subQueryExecState->m_tupleDesc->tdrefcount == -1); + } + } + if (subQueryExecState->m_slot == nullptr) { + MOT_ASSERT(subQueryExecState->m_tupleDesc != nullptr); + MOT_LOG_TRACE("Preparing sub-query %u result slot", i); + subQueryExecState->m_slot = MakeSingleTupleTableSlot(subQueryExecState->m_tupleDesc); + if (subQueryExecState->m_slot == nullptr) { + MOT_LOG_TRACE("Failed to generate sub-query %u tuple table slot", i); + return false; // safe cleanup during destroy + } + Assert(subQueryExecState->m_tupleDesc->tdrefcount == -1); + } + if (subQueryExecState->m_searchKey == nullptr) { + MOT_LOG_TRACE( + "Preparing sub-query %u search key from index %s", i, subQueryContext->m_index->GetName().c_str()); + subQueryExecState->m_searchKey = PrepareJitSearchKey(jitContext, subQueryContext->m_index); + if (subQueryExecState->m_searchKey == nullptr) { + MOT_LOG_TRACE("Failed to generate sub-query %u search key", i); + return false; // safe cleanup during destroy + } + } + if ((subQueryContext->m_commandType == JIT_COMMAND_AGGREGATE_RANGE_SELECT) && + (subQueryExecState->m_endIteratorKey == nullptr)) { + MOT_LOG_TRACE("Preparing sub-query %u end-iterator search key from index %s", + i, + subQueryContext->m_index->GetName().c_str()); + subQueryExecState->m_endIteratorKey = PrepareJitSearchKey(jitContext, subQueryContext->m_index); + if (subQueryExecState->m_endIteratorKey == nullptr) { + MOT_LOG_TRACE("Failed to generate sub-query %u end-iterator search key", i); + return false; // safe cleanup during destroy + } + } + } + return true; +} + +static bool PrepareInvokeContext(JitQueryContext* jitContext, JitQueryExecState* execState) +{ + const char* invokedQueryString = + jitContext->m_invokeContext ? jitContext->m_invokeContext->m_queryString : jitContext->m_invokedQueryString; + MOT_LOG_TRACE("Preparing INVOKE command parameter list for %u parameters into function: %s", + jitContext->m_invokeParamCount, + invokedQueryString); + + // it is possible after revalidation that number of parameter for INVOKE changes + if (jitContext->m_invokeParamCount == 0) { + if (execState->m_invokeParams != nullptr) { + MOT::MemSessionFree(execState->m_invokeParams); + execState->m_invokeParams = nullptr; + } + } else { + if (execState->m_invokeParams && + (execState->m_invokeParams->numParams != (int)jitContext->m_invokeParamCount)) { + MOT::MemSessionFree(execState->m_invokeParams); + execState->m_invokeParams = nullptr; + } + if (execState->m_invokeParams == nullptr) { + execState->m_invokeParams = CreateParamListInfo(jitContext->m_invokeParamCount, false); + if (execState->m_invokeParams == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT", + "Failed to prepare %u invoke parameter", + jitContext->m_invokeParamCount); + return false; // safe cleanup during destroy + } + } + } + MOT_LOG_TRACE("Prepared INVOKE command parameter list %p for %u parameters into function: %s", + execState->m_invokeParams, + jitContext->m_invokeParamCount, + invokedQueryString); + + if (jitContext->m_invokeContext != nullptr) { + if (!PrepareJitContext(jitContext->m_invokeContext)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Execute JIT", "Failed to prepare invoked function context: %s", invokedQueryString); + return false; // safe cleanup during destroy + } + } + return true; +} + +static bool PrepareJitQueryContext(JitQueryContext* jitContext) +{ + MOT_LOG_TRACE("Preparing context %p for query: %s", jitContext, jitContext->m_queryString); + // allocate execution state on-demand + if (jitContext->m_execState == nullptr) { + if (!AllocJitQueryExecState(jitContext)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Execute JIT Query", "Failed to allocate query execution state"); + return false; + } + } + JitQueryExecState* execState = (JitQueryExecState*)jitContext->m_execState; + JitStatisticsProvider::GetInstance().AddSessionBytes((int64_t)sizeof(JitQueryExecState)); + + // re-fetch all table and index objects in case they were removed after TRUNCATE TABLE + if (!RefetchTablesAndIndices(jitContext)) { + MOT_LOG_TRACE("Failed to re-fetch table and index objects"); + return false; // safe cleanup during destroy + } + + jitContext->m_rc = MOT::RC_OK; + // allocate search key (except when executing INSERT or FULL-SCAN SELECT command) + bool allocSearchKey = ((execState->m_searchKey == nullptr) && IsCommandUsingIndex(jitContext->m_commandType) && + (jitContext->m_commandType != JIT_COMMAND_FULL_SELECT)); + if (allocSearchKey) { + if (!PrepareMainSearchKey(jitContext, execState)) { + return false; // safe cleanup during destroy + } + } + + // allocate bitmap-set object for incremental-redo when executing UPDATE command + bool allocBitmapSet = ((execState->m_bitmapSet == nullptr) && + ((jitContext->m_commandType == JIT_COMMAND_UPDATE) || + (jitContext->m_commandType == JIT_COMMAND_RANGE_UPDATE))); + if (allocBitmapSet) { + if (!PrepareUpdateBitmap(jitContext, execState)) { + return false; // safe cleanup during destroy + } + } + + // allocate end-iterator key object when executing range UPDATE command or special SELECT commands + bool allocEndItrKey = ((execState->m_endIteratorKey == nullptr) && IsRangeCommand(jitContext->m_commandType)); + if (allocEndItrKey) { + if (!PrepareEndIteratorKey(jitContext, execState)) { + return false; // safe cleanup during destroy + } + } + + // allocate inner loop search key for JOIN commands + bool allocInnerSearchKey = ((execState->m_innerSearchKey == nullptr) && IsJoinCommand(jitContext->m_commandType)); + if (allocInnerSearchKey) { + if (!PrepareInnerSrearchKey(jitContext, execState)) { + return false; // safe cleanup during destroy + } } // allocate inner loop end-iterator search key for JOIN commands - if ((jitContext->m_innerEndIteratorKey == nullptr) && IsJoinCommand(jitContext->m_commandType)) { - MOT_LOG_TRACE("Preparing inner end iterator key for JOIN command from index %s", - jitContext->m_innerIndex->GetName().c_str()); - jitContext->m_innerEndIteratorKey = PrepareJitSearchKey(jitContext, jitContext->m_innerIndex); - if (jitContext->m_innerEndIteratorKey == nullptr) { - MOT_LOG_TRACE( - "Failed to allocate reusable inner end iterator key for JIT context, aborting jitted code execution"); + bool allocInnerEndItrKey = + ((execState->m_innerEndIteratorKey == nullptr) && IsJoinCommand(jitContext->m_commandType)); + if (allocInnerEndItrKey) { + if (!PrepareInnerEndIteratorKey(jitContext, execState)) { return false; // safe cleanup during destroy } - - MOT_LOG_TRACE("Prepared inner end iterator key %p (%u bytes) for JOIN command from index %s", - jitContext->m_innerEndIteratorKey, - jitContext->m_innerEndIteratorKey->GetKeyLength(), - jitContext->m_innerIndex->GetName().c_str()); } // preparing outer row copy for JOIN commands - if ((jitContext->m_outerRowCopy == nullptr) && IsJoinCommand(jitContext->m_commandType)) { + bool allocOuterRowCopy = ((execState->m_outerRowCopy == nullptr) && IsJoinCommand(jitContext->m_commandType)); + if (allocOuterRowCopy) { MOT_LOG_TRACE("Preparing outer row copy for JOIN command"); - jitContext->m_outerRowCopy = jitContext->m_table->CreateNewRow(); - if (jitContext->m_outerRowCopy == nullptr) { + execState->m_outerRowCopy = jitContext->m_table->CreateNewRow(); + if (execState->m_outerRowCopy == nullptr) { MOT_LOG_TRACE("Failed to allocate reusable outer row copy for JIT context, aborting jitted code execution"); return false; // safe cleanup during destroy } } + // prepare sub-query data for COMPOUND commands + if (jitContext->m_commandType == JIT_COMMAND_COMPOUND_SELECT) { + MemoryContext oldCtx = CurrentMemoryContext; + CurrentMemoryContext = SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR); + if (!PrepareCompoundSubQuery(jitContext, execState)) { + CurrentMemoryContext = oldCtx; + return false; // safe cleanup during destroy + } + CurrentMemoryContext = oldCtx; + } + + if (jitContext->m_commandType == JIT_COMMAND_INVOKE) { + if (!PrepareInvokeContext(jitContext, execState)) { + return false; // safe cleanup during destroy + } + } + return true; } -static bool PrepareJitContextSubQueryData(JitContext* jitContext) +static bool AllocJitFunctionExecState(JitFunctionContext* jitContext) { - // allocate sub-query search keys and generate tuple table slot array using session top memory context - MemoryContext oldCtx = CurrentMemoryContext; - CurrentMemoryContext = SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR); - for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { - JitContext::SubQueryData* subQueryData = &jitContext->m_subQueryData[i]; - if (subQueryData->m_tupleDesc == nullptr) { - MOT_LOG_TRACE("Preparing sub-query %u tuple descriptor", i); - List* targetList = GetSubQueryTargetList(jitContext->m_queryString, i); - if (targetList == nullptr) { - MOT_LOG_TRACE("Failed to locate sub-query %u target list", i); - CurrentMemoryContext = oldCtx; - return false; // safe cleanup during destroy - } + size_t allocSize = sizeof(JitFunctionExecState); + JitFunctionExecState* execState = (JitFunctionExecState*)MOT::MemSessionAlloc(allocSize); + if (execState == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT Query", + "Failed to allocate %u bytes for function execution state", + (unsigned)allocSize); + return false; + } - subQueryData->m_tupleDesc = ExecCleanTypeFromTL(targetList, false); - if (subQueryData->m_tupleDesc == nullptr) { - MOT_LOG_TRACE("Failed to create sub-query %u tuple descriptor from target list", i); - CurrentMemoryContext = oldCtx; - return false; // safe cleanup during destroy - } + errno_t erc = memset_s(execState, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + + // we might have pure SPs (i.e. no queries into MOT tables), so check for count + if (jitContext->m_SPSubQueryCount > 0) { + allocSize = sizeof(JitInvokedQueryExecState) * jitContext->m_SPSubQueryCount; + execState->m_invokedQueryExecState = (JitInvokedQueryExecState*)MOT::MemSessionAlloc(allocSize); + if (execState->m_invokedQueryExecState == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT Query", + "Failed to allocate %u bytes for invoked-queries in function execution state", + (unsigned)allocSize); + MOT::MemSessionFree(execState); + return false; } - if (subQueryData->m_slot == nullptr) { - MOT_ASSERT(subQueryData->m_tupleDesc != nullptr); - MOT_LOG_TRACE("Preparing sub-query %u result slot", i); - subQueryData->m_slot = MakeSingleTupleTableSlot(subQueryData->m_tupleDesc); - if (subQueryData->m_slot == nullptr) { - MOT_LOG_TRACE("Failed to generate sub-query %u tuple table slot", i); - CurrentMemoryContext = oldCtx; - return false; // safe cleanup during destroy - } - } + erc = memset_s(execState->m_invokedQueryExecState, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + } - if (subQueryData->m_searchKey == nullptr) { - MOT_LOG_TRACE( - "Preparing sub-query %u search key from index %s", i, subQueryData->m_index->GetName().c_str()); - subQueryData->m_searchKey = PrepareJitSearchKey(jitContext, subQueryData->m_index); - if (subQueryData->m_searchKey == nullptr) { - MOT_LOG_TRACE("Failed to generate sub-query %u search key", i); - CurrentMemoryContext = oldCtx; - return false; // safe cleanup during destroy - } - } + jitContext->m_execState = (JitExecState*)execState; - if ((subQueryData->m_commandType == JIT_COMMAND_AGGREGATE_RANGE_SELECT) && - (subQueryData->m_endIteratorKey == nullptr)) { - MOT_LOG_TRACE("Preparing sub-query %u end-iterator search key from index %s", - i, - subQueryData->m_index->GetName().c_str()); - subQueryData->m_endIteratorKey = PrepareJitSearchKey(jitContext, subQueryData->m_index); - if (subQueryData->m_endIteratorKey == nullptr) { - MOT_LOG_TRACE("Failed to generate sub-query %u end-iterator search key", i); - CurrentMemoryContext = oldCtx; - return false; // safe cleanup during destroy + return true; +} + +static void GetTypeDescriptor(PLpgSQL_type* typeDesc, Form_pg_attribute attr) +{ + typeDesc->dtype = PLPGSQL_DTYPE_VAR; + typeDesc->dno = 0; // not used, right? + typeDesc->ispkg = false; + typeDesc->ttype = PLPGSQL_TTYPE_SCALAR; + typeDesc->typoid = attr->atttypid; + typeDesc->atttypmod = attr->atttypmod; + typeDesc->collation = attr->attcollation; + typeDesc->typbyval = attr->attbyval; + typeDesc->typlen = attr->attlen; + typeDesc->typname = attr->attname.data; + typeDesc->typrelid = attr->attrelid; + + Oid typeInputOid = InvalidOid; + getTypeInputInfo(attr->atttypid, &typeInputOid, &typeDesc->typioparam); + fmgr_info(typeInputOid, &typeDesc->typinput); +} + +static bool PrepareCallSite( + JitFunctionContext* jitContext, JitInvokedQueryExecState* execState, JitCallSite* callSite, int subQueryId) +{ + // a sub-query result tuple might change, so we regenerate unconditionally work slot and result slot + const char* queryType = callSite->m_queryContext ? "jittable" : "non-jittable"; + const char* queryString = + callSite->m_queryContext ? callSite->m_queryContext->m_queryString : callSite->m_queryString; + if (queryString == nullptr) { + MOT_LOG_TRACE("Invalid call site: missing query string"); + return false; + } + MOT_LOG_TRACE( + "Preparing stored-procedure %s sub-query %u invoke exec state at: %p", queryType, subQueryId, execState); + MOT_LOG_TRACE("Preparing stored-procedure %s sub-query %u work slot: %s", queryType, subQueryId, queryString); + MOT_ASSERT(callSite->m_tupDesc != nullptr); + Assert(callSite->m_tupDesc->tdrefcount == -1); + if (execState->m_workSlot != nullptr) { + execState->m_workSlot->tts_tupleDescriptor = nullptr; + ExecDropSingleTupleTableSlot(execState->m_workSlot); + execState->m_workSlot = nullptr; + } + execState->m_workSlot = MakeSingleTupleTableSlot(callSite->m_tupDesc); + if (execState->m_workSlot == nullptr) { + MOT_LOG_TRACE("Failed to generate stored-procedure %s sub-query %u work tuple table slot: %s", + queryType, + subQueryId, + queryString); + return false; // safe cleanup during destroy + } + Assert(callSite->m_tupDesc->tdrefcount == -1); + + MOT_LOG_TRACE("Preparing stored-procedure %s sub-query %u result slot: %s", queryType, subQueryId, queryString); + if (execState->m_resultSlot != nullptr) { + execState->m_resultSlot->tts_tupleDescriptor = nullptr; + ExecDropSingleTupleTableSlot(execState->m_resultSlot); + execState->m_resultSlot = nullptr; + } + execState->m_resultSlot = MakeSingleTupleTableSlot(callSite->m_tupDesc); + if (execState->m_resultSlot == nullptr) { + MOT_LOG_TRACE("Failed to generate stored-procedure %s sub-query %u result tuple table slot: %s", + queryType, + subQueryId, + queryString); + return false; // safe cleanup during destroy + } + Assert(callSite->m_tupDesc->tdrefcount == -1); + + // a sub-query parameter list might change (in size and/or types), so we regenerate it unconditionally + MOT_LOG_TRACE("Preparing stored-procedure %s sub-query %u parameters list of size %d: %s", + queryType, + subQueryId, + jitContext->m_paramCount, + queryString); + if (execState->m_params != nullptr) { + MOT::MemSessionFree(execState->m_params); + execState->m_params = nullptr; + } + + MOT_LOG_TRACE("Preparing stored-procedure %s sub-query %u param list: %s", queryType, subQueryId, queryString); + execState->m_params = CreateParamListInfo(jitContext->m_paramCount, false); + if (execState->m_params == nullptr) { + MOT_LOG_TRACE("Failed to clone %s sub-query %u parameter list: %s", queryType, subQueryId, queryString); + return false; // safe cleanup during destroy + } + for (int i = 0; i < execState->m_params->numParams; ++i) { + execState->m_params->params[i].isnull = true; + execState->m_params->params[i].ptype = jitContext->m_paramTypes[i]; + execState->m_params->params[i].pflags = 0; + MOT_LOG_DEBUG("param-type: %u", execState->m_params->params[i].ptype); + } + + if (execState->m_resultTypes == nullptr) { + uint32_t attrCount = (uint32_t)execState->m_resultSlot->tts_tupleDescriptor->natts; + if (attrCount > 0) { + uint32_t allocSize = sizeof(PLpgSQL_type) * attrCount; + execState->m_resultTypes = (PLpgSQL_type*)MOT::MemSessionAlloc(allocSize); + if (execState->m_resultTypes == nullptr) { + MOT_LOG_TRACE("Failed to allocate %u bytes for %u result type descriptors", allocSize, attrCount); + return false; + } + for (uint32_t i = 0; i < attrCount; ++i) { + PLpgSQL_type* typeDesc = &execState->m_resultTypes[i]; + Form_pg_attribute attr = &execState->m_resultSlot->tts_tupleDescriptor->attrs[i]; + GetTypeDescriptor(typeDesc, attr); } } } - CurrentMemoryContext = oldCtx; + + // NOTE: JIT context of each sub-query is prepared (in PrepareCallSitePlans) only after the call site plans + // are generated. This will ensure we have necessary locks for the sub-queries. + return true; } -extern bool PrepareJitContext(JitContext* jitContext) +class CallSitePlanGenerator : public JitFunctionQueryVisitor { +public: + explicit CallSitePlanGenerator(JitFunctionContext* jitContext) : m_jitContext(jitContext) + { + BuildExprQueryMap(); + m_queryPlans.resize(m_jitContext->m_SPSubQueryCount, false); + } + + ~CallSitePlanGenerator() final + {} + + JitVisitResult OnQuery(PLpgSQL_expr* expr, PLpgSQL_row* row, int exprIndex, bool into) final + { + // find call site by expression index + int queryIndex = GetQueryIndex(exprIndex); + if (queryIndex == -1) { + MOT_LOG_DEBUG("Cannot find call site for query with expression index %d: %s", exprIndex, expr->query); + return JitVisitResult::JIT_VISIT_CONTINUE; + } + JitCallSite* callSite = &m_jitContext->m_SPSubQueryList[queryIndex]; + JitFunctionExecState* functionExecState = (JitFunctionExecState*)m_jitContext->m_execState; + JitInvokedQueryExecState* execState = &functionExecState->m_invokedQueryExecState[queryIndex]; + const char* queryType = callSite->m_queryContext ? "jittable" : "non-jittable"; + const char* queryString = expr->query; + + if (execState->m_plan != nullptr) { + return JitVisitResult::JIT_VISIT_CONTINUE; + } + + // invoke call site does not need SPI plan + if ((callSite->m_queryContext != nullptr) && (callSite->m_queryContext->m_commandType == JIT_COMMAND_INVOKE)) { + return JitVisitResult::JIT_VISIT_CONTINUE; + } + + MOT_LOG_TRACE("Preparing stored-procedure %s sub-query %u plan: %s", queryType, queryIndex, queryString); + execState->m_plan = GetSpiPlan(functionExecState->m_function, expr); + if (execState->m_plan == nullptr) { + MOT_LOG_TRACE("Failed to prepare SPI plan for %s sub-query %u: %s", queryType, queryIndex, queryString); + return JitVisitResult::JIT_VISIT_ERROR; // safe cleanup during destroy + } + + // setup parameter list + execState->m_expr = expr; + execState->m_params->paramFetch = plpgsql_param_fetch; + execState->m_params->paramFetchArg = &functionExecState->m_estate; + execState->m_params->parserSetup = (ParserSetupHook)plpgsql_parser_setup; + execState->m_params->parserSetupArg = (void*)execState->m_expr; + execState->m_params->params_need_process = false; + + return JitVisitResult::JIT_VISIT_CONTINUE; + } + + void OnError(const char* stmtType, int lineNo) final + { + MOT_LOG_TRACE("Failed to generate call site plan while processing statement at line %d: %s", lineNo, stmtType); + } + + bool AllQueryPlansGenerated() + { + for (uint32_t i = 0; i < m_jitContext->m_SPSubQueryCount; ++i) { + if (!m_queryPlans[i]) { + MOT_LOG_TRACE("Query plan for call site %u not generated", i); + return false; + } + } + return true; + } + +private: + JitFunctionContext* m_jitContext; + using JitExprQueryMap = std::map; + JitExprQueryMap m_exprQueryMap; + std::vector m_queryPlans; + + void BuildExprQueryMap() + { + for (uint32_t i = 0; i < m_jitContext->m_SPSubQueryCount; ++i) { + JitCallSite* callSite = &m_jitContext->m_SPSubQueryList[i]; + m_exprQueryMap.insert(JitExprQueryMap::value_type(callSite->m_exprIndex, (int)i)); + } + } + + int GetQueryIndex(int exprIndex) + { + int queryIndex = -1; + JitExprQueryMap::iterator itr = m_exprQueryMap.find(exprIndex); + if (itr != m_exprQueryMap.end()) { + queryIndex = itr->second; + if (m_queryPlans[queryIndex] == true) { // query hit twice + MOT_LOG_TRACE("Query %d hit twice while generating call site plans", queryIndex); + queryIndex = -1; + } else { + m_queryPlans[queryIndex] = true; + } + } + return queryIndex; + } +}; + +static bool PrepareCallSitePlans(JitFunctionContext* jitContext) { - // allocate argument-is-null array - if (jitContext->m_argIsNull == nullptr) { - MOT_LOG_TRACE("Allocating null argument array with %u slots", (unsigned)jitContext->m_argCount); - jitContext->m_argIsNull = (int*)MOT::MemSessionAlloc(sizeof(int) * jitContext->m_argCount); - if (jitContext->m_argIsNull == nullptr) { - MOT_LOG_TRACE("Failed to allocate null argument array in size of %d slots", jitContext->m_argCount); + CallSitePlanGenerator planGenerator(jitContext); + JitFunctionExecState* functionExecState = (JitFunctionExecState*)jitContext->m_execState; + + SPIAutoConnect spiAutoConnect; + if (!spiAutoConnect.IsConnected()) { + int rc = spiAutoConnect.GetErrorCode(); + MOT_LOG_TRACE("Failed to connect to SPI while generating SPI plans for jitted function: %s", + jitContext->m_queryString, + SPI_result_code_string(rc), + rc); + return false; + } + + if (!VisitFunctionQueries(functionExecState->m_function, &planGenerator)) { + MOT_LOG_TRACE("Failed to generate call site plans: error encountered"); + return false; + } + + if (!planGenerator.AllQueryPlansGenerated()) { + MOT_LOG_TRACE("Failed to generate call site plans: not all plans generated"); + return false; + } + + volatile bool result = true; + volatile CachedPlan* cplan = nullptr; + volatile JitInvokedQueryExecState* invokeExecState = nullptr; + volatile MemoryContext origCxt = CurrentMemoryContext; + PG_TRY(); + { + // Prepare JIT context of each sub-query only after the call site plans are generated. This will ensure we have + // necessary locks for the sub-queries. + for (uint32_t i = 0; i < jitContext->m_SPSubQueryCount; ++i) { + JitCallSite* callSite = &jitContext->m_SPSubQueryList[i]; + // prepare JIT context state of each invoked query + if (callSite->m_queryContext != nullptr) { + MOT_LOG_TRACE("Preparing stored-procedure jittable sub-query %u context: %s", + i, + callSite->m_queryContext->m_queryString); + + // in case we hit parsing during revalidate, we must make sure we have up-to-date parameters + invokeExecState = &functionExecState->m_invokedQueryExecState[i]; + functionExecState->m_estate.cur_expr = invokeExecState->m_expr; // required by plpgsql_param_fetch + if (invokeExecState->m_expr != nullptr) { + invokeExecState->m_expr->func = functionExecState->m_function; + invokeExecState->m_expr->func->cur_estate = (PLpgSQL_execstate*)&functionExecState->m_estate; + } + + if (callSite->m_queryContext->m_commandType != JitExec::JIT_COMMAND_INVOKE) { + cplan = SPI_plan_get_cached_plan(invokeExecState->m_plan); + if (cplan == nullptr) { + MOT_LOG_ERROR("Failed to get cached plan"); + result = false; + break; + } + } + + result = PrepareJitContext(callSite->m_queryContext); + + if (cplan != nullptr) { + ReleaseCachedPlan((CachedPlan*)cplan, invokeExecState->m_plan->saved); + cplan = nullptr; + } + + if (!result) { + MOT_LOG_TRACE("Failed to prepare sub-query %u context", i); + break; // safe cleanup during destroy + } + } + } + + MOT_ASSERT(cplan == nullptr); + } + PG_CATCH(); + { + (void)MemoryContextSwitchTo(origCxt); + + if (cplan != nullptr) { + ReleaseCachedPlan((CachedPlan*)cplan, invokeExecState->m_plan->saved); + cplan = nullptr; + } + + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while preparing sub-query contexts for function %s: %s", + jitContext->m_queryString, + edata->message); + ereport(WARNING, + (errmodule(MOD_MOT), + errmsg("Caught exception while preparing sub-query contexts for function %s: %s", + jitContext->m_queryString, + edata->message), + errdetail("%s", edata->detail))); + FlushErrorState(); + FreeErrorData(edata); + result = false; + } + PG_END_TRY(); + + return result; +} + +static bool PrepareJitFunctionContext(JitFunctionContext* jitContext) +{ + MOT_LOG_TRACE("Preparing context %p for function: %s", jitContext, jitContext->m_queryString); + // allocate execution state on-demand + if (jitContext->m_execState == nullptr) { + if (!AllocJitFunctionExecState(jitContext)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Execute JIT Query", "Failed to allocate function execution state"); + return false; + } + } + JitStatisticsProvider::GetInstance().AddSessionBytes((int64_t)sizeof(JitFunctionExecState)); + + // prepare compiled function + JitFunctionExecState* execState = (JitFunctionExecState*)jitContext->m_execState; + if (execState->m_function == nullptr) { + execState->m_function = GetPGCompiledFunction(jitContext->m_functionOid); + if (execState->m_function == nullptr || execState->m_function->fn_xmin != jitContext->m_functionTxnId) { + bool functionReplaced = false; + if (execState->m_function == nullptr) { + MOT_REPORT_ERROR( + MOT_ERROR_CONCURRENT_MODIFICATION, "Execute JIT Query", "Failed to retrieve PG function"); + } else { + functionReplaced = true; + if (execState->m_function->fn_xmin > jitContext->m_functionTxnId) { + MOT_REPORT_ERROR(MOT_ERROR_CONCURRENT_MODIFICATION, + "Execute JIT Query", + "PG function definition changed from %" PRIu64 " to %" PRIu64 ", JIT context has older version", + jitContext->m_functionTxnId, + execState->m_function->fn_xmin); + } else { + MOT_REPORT_ERROR(MOT_ERROR_CONCURRENT_MODIFICATION, + "Execute JIT Query", + "PG function definition changed from %" PRIu64 " to %" PRIu64 ", JIT context has newer version", + execState->m_function->fn_xmin, + jitContext->m_functionTxnId); + } + } + + // Expire the JIT source only if the function was dropped or the JIT context/source is based on older + // definition of the function. + // It is possible that JIT context/source is already based on the newer definition of the function, but + // this session has not yet seen the newer definition. In this case, GetPGCompiledFunction will retrieve + // the older function, so we should not expire the JIT source. + if (execState->m_function == nullptr || execState->m_function->fn_xmin > jitContext->m_functionTxnId) { + bool expireInvokeQuery = false; + LockJitSource(jitContext->m_jitSource); + if (jitContext->m_jitSource->m_codegenState == JitCodegenState::JIT_CODEGEN_READY) { + expireInvokeQuery = true; + // We are not expiring the JIT source in a transactional manner, so we cannot set the correct + // m_expireTxnId here. But this is fine, we can still avoid premature revalidation attempts using + // m_functionTxnId itself. + SetJitSourceExpired(jitContext->m_jitSource, 0, functionReplaced); + } + UnlockJitSource(jitContext->m_jitSource); + if (expireInvokeQuery) { + MotJitContext* invokeQueryContext = jitContext->m_parentContext; + LockJitSource(invokeQueryContext->m_jitSource); + // We are not expiring the JIT source in a transactional manner, so we cannot set the correct + // m_expireTxnId here. But this is fine, we can still avoid premature revalidation attempts using + // m_functionTxnId itself. + SetJitSourceExpired(invokeQueryContext->m_jitSource, 0, functionReplaced); + UnlockJitSource(invokeQueryContext->m_jitSource); + } + } + + // we can mark this context as invalid outside lock-scope even if there is a race with compilation done + // event, because both will lead to revalidation, so order of events is not important (see plancache.cpp) + InvalidateJitContext(jitContext, 0); + execState->m_function = nullptr; + return false; + } + ++execState->m_function->use_count; + MOT_LOG_TRACE("PrepareJitFunctionContext(): Increased use count of function %p to %lu: %s", + execState->m_function, + execState->m_function->use_count, + jitContext->m_queryString); + } + + // prepare minimal execution state if required + if (execState->m_function != execState->m_estate.func) { + PrepareExecState(&execState->m_estate, execState->m_function); + } + + // prepare call sites + MemoryContext oldCtx = MemoryContextSwitchTo(SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR)); + for (uint32_t i = 0; i < jitContext->m_SPSubQueryCount; ++i) { + JitCallSite* callSite = &jitContext->m_SPSubQueryList[i]; + if (!PrepareCallSite(jitContext, &execState->m_invokedQueryExecState[i], callSite, i)) { + (void)MemoryContextSwitchTo(oldCtx); return false; } } - // re-fetch all index objects in case they were removed after TRUNCATE TABLE - if (jitContext->m_commandType != JIT_COMMAND_INSERT) { - if (!ReFetchIndices(jitContext)) { - return false; // safe cleanup during destroy - } + if (!PrepareCallSitePlans(jitContext)) { + (void)MemoryContextSwitchTo(oldCtx); + return false; } - // allocate search key (except when executing INSERT or FULL-SCAN SELECT command) - if ((jitContext->m_searchKey == nullptr) && (jitContext->m_commandType != JIT_COMMAND_INSERT) && - (jitContext->m_commandType != JIT_COMMAND_FULL_SELECT)) { - MOT_LOG_TRACE("Preparing search key from index %s", jitContext->m_index->GetName().c_str()); - jitContext->m_searchKey = PrepareJitSearchKey(jitContext, jitContext->m_index); - if (jitContext->m_searchKey == nullptr) { - MOT_LOG_TRACE("Failed to allocate reusable search key for JIT context, aborting jitted code execution"); - return false; // safe cleanup during destroy - } - - MOT_LOG_TRACE("Prepared search key %p (%u bytes) from index %s", - jitContext->m_searchKey, - jitContext->m_searchKey->GetKeyLength(), - jitContext->m_index->GetName().c_str()); - } - - // allocate bitmap-set object for incremental-redo when executing UPDATE command - if ((jitContext->m_bitmapSet == nullptr) && ((jitContext->m_commandType == JIT_COMMAND_UPDATE) || - (jitContext->m_commandType == JIT_COMMAND_RANGE_UPDATE))) { - int fieldCount = (int)jitContext->m_table->GetFieldCount(); - MOT_LOG_TRACE( - "Initializing reusable bitmap set according to %d fields (including null-bits column 0) in table %s", - fieldCount, - jitContext->m_table->GetLongTableName().c_str()); - void* buf = MOT::MemSessionAlloc(sizeof(MOT::BitmapSet)); - if (buf == nullptr) { - MOT_LOG_TRACE("Failed to allocate reusable bitmap set for JIT context, aborting jitted code execution"); - return false; // safe cleanup during destroy - } - - uint8_t* bitmapData = (uint8_t*)MOT::MemSessionAlloc(MOT::BitmapSet::GetLength(fieldCount)); - if (bitmapData == nullptr) { - MOT_LOG_TRACE("Failed to allocate reusable bitmap set for JIT context, aborting jitted code execution"); - MOT::MemSessionFree(buf); - return false; // safe cleanup during destroy - } - jitContext->m_bitmapSet = new (buf) MOT::BitmapSet(bitmapData, fieldCount); - } - - // allocate end-iterator key object when executing range UPDATE command or special SELECT commands - if ((jitContext->m_endIteratorKey == nullptr) && IsRangeCommand(jitContext->m_commandType)) { - MOT_LOG_TRACE("Preparing end iterator key for range update/select command from index %s", - jitContext->m_index->GetName().c_str()); - jitContext->m_endIteratorKey = PrepareJitSearchKey(jitContext, jitContext->m_index); - if (jitContext->m_endIteratorKey == nullptr) { - MOT_LOG_TRACE( - "Failed to allocate reusable end iterator key for JIT context, aborting jitted code execution"); - return false; // safe cleanup during destroy - } - - MOT_LOG_TRACE("Prepared end iterator key %p (%u bytes) for range update/select command from index %s", - jitContext->m_endIteratorKey, - jitContext->m_endIteratorKey->GetKeyLength(), - jitContext->m_index->GetName().c_str()); - } - - if (!PrepareJitContextJoinData(jitContext)) { - MOT_LOG_TRACE("Failed to allocate join related data for JIT context, aborting jitted code execution"); - return false; // safe cleanup during destroy - } - - // prepare sub-query data for COMPOUND commands - if (jitContext->m_commandType == JIT_COMMAND_COMPOUND_SELECT) { - if (!PrepareJitContextSubQueryData(jitContext)) { - MOT_LOG_TRACE("Failed to sub-query data for JIT context, aborting jitted code execution"); - return false; // safe cleanup during destroy - } - } + (void)MemoryContextSwitchTo(oldCtx); return true; } @@ -560,235 +1784,1963 @@ static void DestroyDatumArray(JitDatumArray* datumArray, JitContextUsage usage) MOT_ASSERT(datumArray->m_datums != nullptr); for (uint32_t i = 0; i < datumArray->m_datumCount; ++i) { if (!datumArray->m_datums[i].m_isNull && !IsPrimitiveType(datumArray->m_datums[i].m_type)) { - if (usage == JIT_CONTEXT_GLOBAL) { - MOT::MemGlobalFree(DatumGetPointer(datumArray->m_datums[i].m_datum)); - } else { - MOT::MemSessionFree(DatumGetPointer(datumArray->m_datums[i].m_datum)); - } + JitMemFree(DatumGetPointer(datumArray->m_datums[i].m_datum), usage); } } - if (usage == JIT_CONTEXT_GLOBAL) { - MOT::MemGlobalFree(datumArray->m_datums); - } else { - MOT::MemSessionFree(datumArray->m_datums); - } + JitMemFree(datumArray->m_datums, usage); datumArray->m_datums = nullptr; datumArray->m_datumCount = 0; } + + MOT_ASSERT(datumArray->m_datums == nullptr); } -extern void DestroyJitContext(JitContext* jitContext) +extern void DestroyJitContext(MotJitContext* jitContext, bool isDropCachedPlan /* = false */) +{ + if (jitContext == nullptr) { + return; + } + + if (isDropCachedPlan) { + (void)EnsureSafeThreadAccess(); + } + +#ifdef MOT_JIT_DEBUG + MOT_LOG_TRACE("Destroying JIT context %p with %" PRIu64 " executions of query: %s", + jitContext, + jitContext->m_execState ? jitContext->m_execState->m_execCount : 0, + jitContext->m_queryString); +#else + MOT_LOG_TRACE("Destroying %s JIT %s context %p of query: %s", + JitContextUsageToString(jitContext->m_usage), + JitContextTypeToString(jitContext->m_contextType), + jitContext, + jitContext->m_queryString); +#endif + + // remove from JIT source + if (jitContext->m_jitSource != nullptr) { + RemoveJitSourceContext(jitContext->m_jitSource, jitContext); + jitContext->m_jitSource = nullptr; + } + + // cleanup constant datum array + DestroyDatumArray(&jitContext->m_constDatums, jitContext->m_usage); + + // cleanup + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + DestroyJitQueryContext((JitQueryContext*)jitContext, isDropCachedPlan); + } else { + DestroyJitFunctionContext((JitFunctionContext*)jitContext, isDropCachedPlan); + } + + // cleanup code generator (only in global-usage, not even global-secondary usage) + if ((jitContext->m_codeGen != nullptr) && (jitContext->m_usage == JIT_CONTEXT_GLOBAL)) { + FreeGsCodeGen(jitContext->m_codeGen); + jitContext->m_codeGen = nullptr; + } + + // return context to pool + FreeJitContext(jitContext); +} + +static void DestroyJitNonNativeSortExecState( + JitNonNativeSortExecState* jitNonNativeSortExecState, JitContextUsage usage) +{ + MOT_ASSERT(jitNonNativeSortExecState); + + if (jitNonNativeSortExecState->m_tupleSort) { + tuplesort_end((Tuplesortstate*)jitNonNativeSortExecState->m_tupleSort); + jitNonNativeSortExecState->m_tupleSort = nullptr; + } + + JitMemFree(jitNonNativeSortExecState, usage); +} + +static void DestroyJitExecState(JitExecState* execState) +{ + if (execState != nullptr) { + MOT::MemSessionFree(execState); + } +} + +static void CleanupExecStatePrimary(JitQueryContext* jitContext, JitQueryExecState* execState, bool isDropCachedPlan) +{ + if (execState == nullptr) { + return; + } + + if (jitContext->m_table == nullptr) { + return; + } + + MOT::Table* table = nullptr; + if (isDropCachedPlan) { + // If this is called from DropCachedPlan, lock the table. Because for deallocate and deallocate all, there is + // no table locks taken in the envelope, so we must lock the MOT::Table before destroying keys and rows. + if (u_sess->mot_cxt.txn_manager != nullptr) { + table = u_sess->mot_cxt.txn_manager->GetTxnTable(jitContext->m_tableId); + if (table != nullptr) { + table->RdLock(); + } + } else { + table = MOT::GetTableManager()->GetTableSafeByExId(jitContext->m_tableId); + } + if (table == nullptr) { + // Table is dropped concurrently, just set everything to nullptr and return. + execState->m_searchKey = nullptr; + execState->m_endIteratorKey = nullptr; + execState->m_beginIterator = nullptr; + execState->m_endIterator = nullptr; + execState->m_outerRowCopy = nullptr; + return; + } + MOT_ASSERT(table == jitContext->m_table); + } + + if (jitContext->m_index != nullptr) { + MOT::Index* index = jitContext->m_index; + if (table != nullptr) { + index = table->GetIndexByExtId(jitContext->m_indexId); + } else { + index = jitContext->m_table->GetIndexByExtId(jitContext->m_indexId); + } + + if (jitContext->m_index == index) { + // Destroy the keys and iterators only if the index pointer has not changed. + if (execState->m_searchKey != nullptr) { + index->DestroyKey(execState->m_searchKey); + } + + if (execState->m_endIteratorKey != nullptr) { + index->DestroyKey(execState->m_endIteratorKey); + } + + if (execState->m_beginIterator != nullptr) { + destroyIterator(execState->m_beginIterator); + } + + if (execState->m_endIterator != nullptr) { + destroyIterator(execState->m_endIterator); + } + } + + execState->m_searchKey = nullptr; + execState->m_endIteratorKey = nullptr; + execState->m_beginIterator = nullptr; + execState->m_endIterator = nullptr; + } + + // cleanup JOIN outer row copy + if (execState->m_outerRowCopy != nullptr) { + jitContext->m_table->DestroyRow(execState->m_outerRowCopy); + execState->m_outerRowCopy = nullptr; + } + + if (table != nullptr) { + table->Unlock(); + } +} + +static void CleanupExecStateInner(JitQueryContext* jitContext, JitQueryExecState* execState, bool isDropCachedPlan) +{ + if (execState == nullptr) { + return; + } + + if (jitContext->m_innerTable == nullptr) { + return; + } + + MOT::Table* table = nullptr; + if (isDropCachedPlan) { + // If this is called from DropCachedPlan, lock the table. Because for deallocate and deallocate all, there is + // no table locks taken in the envelope, so we must lock the MOT::Table before destroying keys and rows. + if (u_sess->mot_cxt.txn_manager != nullptr) { + table = u_sess->mot_cxt.txn_manager->GetTxnTable(jitContext->m_innerTableId); + if (table != nullptr) { + table->RdLock(); + } + } else { + table = MOT::GetTableManager()->GetTableSafeByExId(jitContext->m_innerTableId); + } + if (table == nullptr) { + // Table is dropped concurrently, just set everything to nullptr and return. + execState->m_innerSearchKey = nullptr; + execState->m_innerEndIteratorKey = nullptr; + execState->m_innerBeginIterator = nullptr; + execState->m_innerEndIterator = nullptr; + execState->m_innerRow = nullptr; + return; + } + MOT_ASSERT(table == jitContext->m_innerTable); + } + + if (jitContext->m_innerIndex != nullptr) { + MOT::Index* index = jitContext->m_innerIndex; + if (table != nullptr) { + index = table->GetIndexByExtId(jitContext->m_innerIndexId); + } else { + index = jitContext->m_innerTable->GetIndexByExtId(jitContext->m_innerIndexId); + } + + if (jitContext->m_innerIndex == index) { + // Destroy the keys and iterators only if the index pointer has not changed. + if (execState->m_innerSearchKey != nullptr) { + index->DestroyKey(execState->m_innerSearchKey); + } + + if (execState->m_innerEndIteratorKey != nullptr) { + index->DestroyKey(execState->m_innerEndIteratorKey); + } + + if (execState->m_innerBeginIterator != nullptr) { + destroyIterator(execState->m_innerBeginIterator); + } + + if (execState->m_innerEndIterator != nullptr) { + destroyIterator(execState->m_innerEndIterator); + } + } + + execState->m_innerSearchKey = nullptr; + execState->m_innerEndIteratorKey = nullptr; + execState->m_innerBeginIterator = nullptr; + execState->m_innerEndIterator = nullptr; + } + + // cleanup JOIN inner row + if (execState->m_innerRow != nullptr) { + jitContext->m_innerTable->DestroyRow(execState->m_innerRow); + execState->m_innerRow = nullptr; + } + + if (table != nullptr) { + table->Unlock(); + } +} + +static void CleanupSubQueryExecState( + JitSubQueryContext* subQueryContext, JitSubQueryExecState* subQueryExecState, bool isDropCachedPlan) +{ + if (subQueryExecState->m_slot != nullptr) { + ExecDropSingleTupleTableSlot(subQueryExecState->m_slot); + subQueryExecState->m_slot = nullptr; + } + + if (subQueryExecState->m_tupleDesc != nullptr) { + FreeTupleDesc(subQueryExecState->m_tupleDesc); + subQueryExecState->m_tupleDesc = nullptr; + } + + if (subQueryContext->m_table == nullptr) { + return; + } + + if (subQueryContext->m_index == nullptr) { + return; + } + + MOT::Table* table = nullptr; + if (isDropCachedPlan) { + // If this is called from DropCachedPlan, lock the table. Because for deallocate and deallocate all, there is + // no table locks taken in the envelope, so we must lock the MOT::Table before destroying keys. + if (u_sess->mot_cxt.txn_manager != nullptr) { + table = u_sess->mot_cxt.txn_manager->GetTxnTable(subQueryContext->m_tableId); + if (table != nullptr) { + table->RdLock(); + } + } else { + table = MOT::GetTableManager()->GetTableSafeByExId(subQueryContext->m_tableId); + } + if (table == nullptr) { + // Table is dropped concurrently, just set everything to nullptr and return. + subQueryExecState->m_searchKey = nullptr; + subQueryExecState->m_endIteratorKey = nullptr; + return; + } + MOT_ASSERT(table == subQueryContext->m_table); + } + + MOT::Index* index = subQueryContext->m_index; + if (table != nullptr) { + index = table->GetIndexByExtId(subQueryContext->m_indexId); + } else { + index = subQueryContext->m_table->GetIndexByExtId(subQueryContext->m_indexId); + } + + if (subQueryContext->m_index == index) { + // Destroy the keys only if the index pointer has not changed. + if (subQueryExecState->m_searchKey != nullptr) { + index->DestroyKey(subQueryExecState->m_searchKey); + } + + if (subQueryExecState->m_endIteratorKey != nullptr) { + index->DestroyKey(subQueryExecState->m_endIteratorKey); + } + } + + subQueryExecState->m_searchKey = nullptr; + subQueryExecState->m_endIteratorKey = nullptr; + + if (table != nullptr) { + table->Unlock(); + } +} + +static void DestroyQueryExecState(JitQueryContext* jitContext, bool isDropCachedPlan = false) +{ + JitQueryExecState* execState = (JitQueryExecState*)jitContext->m_execState; + if (execState == nullptr) { + return; + } + + if ((jitContext->m_subQueryContext != nullptr) && (execState->m_subQueryExecState)) { + for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { + JitSubQueryContext* subQueryContext = &jitContext->m_subQueryContext[i]; + CleanupSubQueryExecState(subQueryContext, &execState->m_subQueryExecState[i], isDropCachedPlan); + } + } + + CleanupExecStatePrimary(jitContext, execState, isDropCachedPlan); + + CleanupExecStateInner(jitContext, execState, isDropCachedPlan); + + // cleanup bitmap set + if (execState->m_bitmapSet != nullptr) { + MOT::MemSessionFree(execState->m_bitmapSet->GetData()); + execState->m_bitmapSet->MOT::BitmapSet::~BitmapSet(); + MOT::MemSessionFree(execState->m_bitmapSet); + execState->m_bitmapSet = nullptr; + } + + // cleanup aggregate array + if (execState->m_aggExecState != nullptr) { + MOT::MemSessionFree(execState->m_aggExecState); + execState->m_aggExecState = nullptr; + } + + // cleanup sub-query array + if (execState->m_subQueryExecState != nullptr) { + MOT::MemSessionFree(execState->m_subQueryExecState); + execState->m_subQueryExecState = nullptr; + } + + // cleanup invoke parameters + if (execState->m_invokeParams != nullptr) { + MOT_ASSERT(!IsJitContextUsageGlobal(jitContext->m_usage)); + JitMemFree(execState->m_invokeParams, jitContext->m_usage); + execState->m_invokeParams = nullptr; + } + + if (execState->m_nonNativeSortExecState != nullptr) { + DestroyJitNonNativeSortExecState(execState->m_nonNativeSortExecState, jitContext->m_usage); + execState->m_nonNativeSortExecState = nullptr; + } + + DestroyJitExecState(execState); + jitContext->m_execState = nullptr; + JitStatisticsProvider::GetInstance().AddSessionBytes((int64_t)sizeof(JitQueryExecState)); +} + +static void DestroyJitQueryContext(JitQueryContext* jitContext, bool isDropCachedPlan /* = false */) +{ + // cleanup execution state + DestroyQueryExecState(jitContext, isDropCachedPlan); + + // cleanup sub-query data array + CleanupJitSubQueryContextArray(jitContext, isDropCachedPlan); + + // cleanup keys(s) + CleanupJitContextPrimary(jitContext, isDropCachedPlan); + + // cleanup JOIN keys(s) + CleanupJitContextInner(jitContext, isDropCachedPlan); + + // cleanup parameter info array + if (jitContext->m_invokeParamInfo != nullptr) { + JitMemFree(jitContext->m_invokeParamInfo, jitContext->m_usage); + jitContext->m_invokeParamInfo = nullptr; + } + + // cleanup invoke context + if (jitContext->m_invokeContext != nullptr) { + DestroyJitContext(jitContext->m_invokeContext, isDropCachedPlan); + jitContext->m_invokeContext = nullptr; + } + + if (jitContext->m_nonNativeSortParams) { + DestroyJitNonNativeSortParams(jitContext->m_nonNativeSortParams, jitContext->m_usage); + jitContext->m_nonNativeSortParams = nullptr; + } +} + +static void DestroyCallSite(JitCallSite* callSite, JitContextUsage usage, bool isDropCachedPlan = false) +{ + if (callSite->m_queryContext != nullptr) { + DestroyJitContext(callSite->m_queryContext, isDropCachedPlan); + callSite->m_queryContext = nullptr; + } + + if (callSite->m_queryString != nullptr) { + JitMemFree(callSite->m_queryString, usage); + callSite->m_queryString = nullptr; + } + + if (callSite->m_callParamInfo != nullptr) { + JitMemFree(callSite->m_callParamInfo, usage); + callSite->m_callParamInfo = nullptr; + } + + if (callSite->m_tupDesc != nullptr) { + FreeTupleDesc(callSite->m_tupDesc); + callSite->m_tupDesc = nullptr; + } +} + +static void DestroyJitInvokeExecState(JitInvokedQueryExecState* execState) +{ + if (execState->m_workSlot != nullptr) { + execState->m_workSlot->tts_tupleDescriptor = nullptr; + ExecDropSingleTupleTableSlot(execState->m_workSlot); + execState->m_workSlot = nullptr; + } + if (execState->m_resultSlot != nullptr) { + execState->m_resultSlot->tts_tupleDescriptor = nullptr; + ExecDropSingleTupleTableSlot(execState->m_resultSlot); + execState->m_resultSlot = nullptr; + } + if (execState->m_params != nullptr) { + MOT::MemSessionFree(execState->m_params); + execState->m_params = nullptr; + } + if (execState->m_plan != nullptr) { + (void)SPI_freeplan(execState->m_plan); + execState->m_plan = nullptr; + } + if (execState->m_resultTypes != nullptr) { + MOT::MemSessionFree(execState->m_resultTypes); + execState->m_resultTypes = nullptr; + } + if (execState->m_expr != nullptr) { + execState->m_expr->func = nullptr; + execState->m_expr = nullptr; + } +} + +static void DestroyFunctionExecState(JitFunctionContext* jitContext) +{ + JitFunctionExecState* execState = (JitFunctionExecState*)jitContext->m_execState; + if (execState != nullptr) { + // release compiled function + if (execState->m_function != nullptr) { + MOT_ASSERT(execState->m_function->use_count > 0); + --execState->m_function->use_count; + MOT_LOG_TRACE("DestroyFunctionExecState(): Decreased use count of function %p to %lu: %s", + execState->m_function, + execState->m_function->use_count, + jitContext->m_queryString); + execState->m_function = nullptr; + } + + // cleanup sub-query invoke ExecState + if (execState->m_invokedQueryExecState != nullptr) { + for (uint32_t i = 0; i < jitContext->m_SPSubQueryCount; ++i) { + DestroyJitInvokeExecState(&execState->m_invokedQueryExecState[i]); + } + MOT::MemSessionFree(execState->m_invokedQueryExecState); + execState->m_invokedQueryExecState = nullptr; + } + + // cleanup common attributes + DestroyJitExecState(execState); + jitContext->m_execState = nullptr; + JitStatisticsProvider::GetInstance().AddSessionBytes((int64_t)sizeof(JitFunctionExecState)); + } +} + +static void DestroyJitFunctionContext(JitFunctionContext* jitContext, bool isDropCachedPlan /* = false */) +{ + // cleanup execution state + DestroyFunctionExecState(jitContext); + + // cleanup sub-query data and tuple descriptors + if (jitContext->m_SPSubQueryList != nullptr) { + for (uint32_t i = 0; i < jitContext->m_SPSubQueryCount; ++i) { + JitCallSite* callSite = &jitContext->m_SPSubQueryList[i]; + DestroyCallSite(callSite, jitContext->m_usage, isDropCachedPlan); + } + + JitMemFree(jitContext->m_SPSubQueryList, jitContext->m_usage); + } + + // cleanup type list + if (jitContext->m_paramTypes != nullptr) { + JitMemFree(jitContext->m_paramTypes, jitContext->m_usage); + jitContext->m_paramTypes = nullptr; + } + + // cleanup result tuple descriptor + if (jitContext->m_resultTupDesc != nullptr) { + FreeTupleDesc(jitContext->m_resultTupDesc); + jitContext->m_resultTupDesc = nullptr; + } + if (jitContext->m_rowTupDesc) { + FreeTupleDesc(jitContext->m_rowTupDesc); + jitContext->m_rowTupDesc = nullptr; + } +} + +static bool JitQueryContextRefersRelation(JitQueryContext* jitContext, uint64_t relationId) +{ + bool result = false; + if (jitContext->m_tableId == relationId) { + result = true; + } else if (jitContext->m_indexId == relationId) { + result = true; + } else if (jitContext->m_innerTableId == relationId) { + result = true; + } else if (jitContext->m_innerIndexId == relationId) { + result = true; + } else if (jitContext->m_subQueryCount > 0) { + for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { + if (jitContext->m_subQueryContext[i].m_tableId == relationId) { + result = true; + break; + } + if (jitContext->m_subQueryContext[i].m_indexId == relationId) { + result = true; + break; + } + } + } else if ((jitContext->m_commandType == JIT_COMMAND_INVOKE) && (jitContext->m_invokeContext != nullptr)) { + result = JitFunctionContextRefersRelation(jitContext->m_invokeContext, relationId); + } + + return result; +} + +static bool JitFunctionContextRefersRelation(JitFunctionContext* jitContext, uint64_t relationId) +{ + bool result = false; + for (uint32_t i = 0; i < jitContext->m_SPSubQueryCount; ++i) { + JitCallSite* callSite = &jitContext->m_SPSubQueryList[i]; + if ((callSite->m_queryContext != nullptr) && + JitQueryContextRefersRelation((JitQueryContext*)callSite->m_queryContext, relationId)) { + result = true; + break; + } + // non-jittable sub-queries are irrelevant in this context + } + return result; +} + +extern bool JitContextRefersRelation(MotJitContext* jitContext, uint64_t relationId) +{ + bool result = false; + if (jitContext != nullptr) { + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + result = JitQueryContextRefersRelation((JitQueryContext*)jitContext, relationId); + } else { + result = JitFunctionContextRefersRelation((JitFunctionContext*)jitContext, relationId); + } + } + return result; +} + +extern void PurgeJitContext(MotJitContext* jitContext, uint64_t relationId) { if (jitContext != nullptr) { #ifdef MOT_JIT_DEBUG - MOT_LOG_TRACE("Destroying JIT context %p with %" PRIu64 " executions of query: %s", + MOT_LOG_TRACE("Purging %s JIT %s context %p by external table %" PRIu64 " with %" PRIu64 + " executions of query: %s", + JitContextUsageToString(jitContext->m_usage), + JitContextTypeToString(jitContext->m_contextType), jitContext, - jitContext->m_execCount, + relationId, + jitContext->m_execState ? jitContext->m_execState->m_execCount : 0, jitContext->m_queryString); #else - MOT_LOG_TRACE("Destroying %s JIT context %p of query: %s", - jitContext->m_usage == JIT_CONTEXT_GLOBAL ? "global" : "session-local", + MOT_LOG_TRACE("Purging %s JIT %s context %p by external table %" PRIu64 " of query: %s", + JitContextUsageToString(jitContext->m_usage), + JitContextTypeToString(jitContext->m_contextType), jitContext, + relationId, jitContext->m_queryString); #endif - // remove from JIT source - if (jitContext->m_jitSource != nullptr) { - RemoveJitSourceContext(jitContext->m_jitSource, jitContext); - jitContext->m_jitSource = nullptr; + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + PurgeJitQueryContext((JitQueryContext*)jitContext, relationId); + } else { + PurgeJitFunctionContext((JitFunctionContext*)jitContext, relationId); + } + if (jitContext->m_execState != nullptr) { + MOT_ATOMIC_STORE(jitContext->m_execState->m_purged, true); + } + } +} + +extern void DeprecateJitContext(MotJitContext* jitContext) +{ + MOT_LOG_TRACE("Deprecating %s JIT context %p: %s", + JitContextUsageToString(jitContext->m_usage), + jitContext, + jitContext->m_queryString); + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + validState |= JIT_CONTEXT_DEPRECATE; + MOT_ATOMIC_STORE(jitContext->m_validState, validState); + + PropagateValidState(jitContext, 0, 0); +} + +extern bool IsJitContextValid(MotJitContext* jitContext) +{ + return (MOT_ATOMIC_LOAD(jitContext->m_validState) == JIT_CONTEXT_VALID); +} + +extern bool IsJitSubContext(MotJitContext* jitContext) +{ + return IsJitSubContextInline(jitContext); +} + +extern bool IsJitContextPendingCompile(MotJitContext* jitContext) +{ + return ((MOT_ATOMIC_LOAD(jitContext->m_validState) & JIT_CONTEXT_PENDING_COMPILE) == JIT_CONTEXT_PENDING_COMPILE); +} + +extern bool IsJitContextDoneCompile(MotJitContext* jitContext) +{ + return ((MOT_ATOMIC_LOAD(jitContext->m_validState) & JIT_CONTEXT_DONE_COMPILE) == JIT_CONTEXT_DONE_COMPILE); +} + +extern bool IsJitContextErrorCompile(MotJitContext* jitContext) +{ + return ((MOT_ATOMIC_LOAD(jitContext->m_validState) & JIT_CONTEXT_ERROR_COMPILE) == JIT_CONTEXT_ERROR_COMPILE); +} + +extern bool GetJitContextCompileState(MotJitContext* jitContext, bool* isPending, bool* isDone, bool* isError) +{ + bool result = false; + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + *isPending = ((validState & JIT_CONTEXT_PENDING_COMPILE) == JIT_CONTEXT_PENDING_COMPILE); + *isDone = ((validState & JIT_CONTEXT_DONE_COMPILE) == JIT_CONTEXT_DONE_COMPILE); + *isError = ((validState & JIT_CONTEXT_ERROR_COMPILE) == JIT_CONTEXT_ERROR_COMPILE); + if (*isPending || *isDone || *isError) { + result = true; + } + return result; +} + +inline const char* JitContextStateToString(JitContextState state) +{ + switch (state) { + case JIT_CONTEXT_STATE_INIT: + return "INIT"; + case JIT_CONTEXT_STATE_READY: + return "READY"; + case JIT_CONTEXT_STATE_PENDING: + return "PENDING"; + case JIT_CONTEXT_STATE_DONE: + return "DONE"; + case JIT_CONTEXT_STATE_ERROR: + return "ERROR"; + case JIT_CONTEXT_STATE_INVALID: + return "INVALID"; + case JIT_CONTEXT_STATE_FINAL: + return "FINAL"; + default: + return "N/A"; + } +} + +extern JitContextState GetJitContextState(MotJitContext* jitContext) +{ + JitContextState state = JIT_CONTEXT_STATE_INIT; + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + if (validState == JIT_CONTEXT_VALID) { + state = JIT_CONTEXT_STATE_READY; + } else if ((validState & JIT_CONTEXT_PENDING_COMPILE) == JIT_CONTEXT_PENDING_COMPILE) { + state = JIT_CONTEXT_STATE_PENDING; + } else if ((validState & JIT_CONTEXT_DONE_COMPILE) == JIT_CONTEXT_DONE_COMPILE) { + state = JIT_CONTEXT_STATE_DONE; + } else if ((validState & JIT_CONTEXT_ERROR_COMPILE) == JIT_CONTEXT_ERROR_COMPILE) { + state = JIT_CONTEXT_STATE_ERROR; + } else if ((validState & JIT_CONTEXT_INVALID) == JIT_CONTEXT_INVALID) { + state = JIT_CONTEXT_STATE_INVALID; + } else if ((validState & JIT_CONTEXT_CHILD_QUERY_INVALID) == JIT_CONTEXT_CHILD_QUERY_INVALID) { + state = JIT_CONTEXT_STATE_INVALID; + } else if ((validState & JIT_CONTEXT_CHILD_SP_INVALID) == JIT_CONTEXT_CHILD_SP_INVALID) { + state = JIT_CONTEXT_STATE_INVALID; + } else if ((validState & JIT_CONTEXT_RELATION_INVALID) == JIT_CONTEXT_RELATION_INVALID) { + state = JIT_CONTEXT_STATE_INVALID; + } else if ((validState & JIT_CONTEXT_DEPRECATE) == JIT_CONTEXT_DEPRECATE) { + state = JIT_CONTEXT_STATE_INVALID; + } + MOT_LOG_TRACE("JIT context %p state is %s (validState %u): %s", + jitContext, + JitContextStateToString(state), + validState, + jitContext->m_queryString); + MOT_ASSERT((state != JIT_CONTEXT_STATE_INIT) && (state != JIT_CONTEXT_STATE_FINAL)); + return state; +} + +inline void MarkJitContextPendingCompile(MotJitContext* jitContext) +{ + // set pending state, be careful not to overwrite done or error state + uint8_t currState = MOT_ATOMIC_LOAD(jitContext->m_validState); + if (((currState & JIT_CONTEXT_DONE_COMPILE) == 0) && ((currState & JIT_CONTEXT_ERROR_COMPILE) == 0)) { + uint8_t newState = currState; + newState &= ~JIT_CONTEXT_DONE_COMPILE; + newState &= ~JIT_CONTEXT_ERROR_COMPILE; + newState |= JIT_CONTEXT_PENDING_COMPILE; + // we don't care if we fail here, because done and error states take precedence over pending state + MOT_ATOMIC_CAS(jitContext->m_validState, currState, newState); + } +} + +extern void MarkJitContextDoneCompile(MotJitContext* jitContext) +{ + // reset pending state, and set done state (it is ok to overwrite pending state) + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + validState &= ~JIT_CONTEXT_PENDING_COMPILE; + validState &= ~JIT_CONTEXT_ERROR_COMPILE; + validState |= JIT_CONTEXT_DONE_COMPILE; + MOT_ATOMIC_STORE(jitContext->m_validState, validState); +} + +extern void MarkJitContextErrorCompile(MotJitContext* jitContext) +{ + // reset pending state, and set done state (it is ok to overwrite pending state) + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + validState &= ~JIT_CONTEXT_PENDING_COMPILE; + validState &= ~JIT_CONTEXT_DONE_COMPILE; + validState |= JIT_CONTEXT_ERROR_COMPILE; + MOT_ATOMIC_STORE(jitContext->m_validState, validState); +} + +extern void ResetJitContextCompileState(MotJitContext* jitContext) +{ + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + validState &= ~JIT_CONTEXT_PENDING_COMPILE; + validState &= ~JIT_CONTEXT_DONE_COMPILE; + validState &= ~JIT_CONTEXT_ERROR_COMPILE; + MOT_ATOMIC_STORE(jitContext->m_validState, validState); +} + +extern void ResetErrorState(JitExecState* execState) +{ + execState->m_errorMessage = PointerGetDatum(nullptr); + execState->m_errorDetail = PointerGetDatum(nullptr); + execState->m_errorHint = PointerGetDatum(nullptr); + execState->m_sqlStateString = PointerGetDatum(nullptr); + execState->m_sqlState = 0; + execState->m_nullColumnId = 0; +} + +static inline void ResetJitContextTable(MotJitContext* jitContext, uint64_t relationId) +{ + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + JitQueryContext* queryContext = (JitQueryContext*)jitContext; + if ((queryContext->m_table != nullptr) && (queryContext->m_tableId == relationId)) { + queryContext->m_table = nullptr; + } + if ((queryContext->m_innerTable != nullptr) && (queryContext->m_innerTableId == relationId)) { + queryContext->m_innerTable = nullptr; + } + } +} + +static void PropagateValidState(MotJitContext* jitContext, uint8_t invalidFlag, uint64_t relationId) +{ + MOT_LOG_TRACE("Propagating valid state of context %p: %s", jitContext, jitContext->m_queryString); + if (invalidFlag == 0) { + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + invalidFlag = JIT_CONTEXT_CHILD_QUERY_INVALID; + } else { + invalidFlag = JIT_CONTEXT_CHILD_SP_INVALID; + } + } + MOT_LOG_TRACE("Valid-state is: %s", JitContextValidStateToString(invalidFlag)); + + uint8_t newValidState = 0; + MotJitContext* parentContext = jitContext->m_parentContext; + while (parentContext != nullptr) { + MOT_LOG_TRACE("Invalidating %s parent JIT context %p by child %s: %s", + JitContextUsageToString(jitContext->m_usage), + parentContext, + (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) ? "query" : "SP", + parentContext->m_queryString); + newValidState = MOT_ATOMIC_LOAD(parentContext->m_validState) | invalidFlag; + MOT_ATOMIC_STORE(parentContext->m_validState, newValidState); + if (parentContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + if (relationId != 0) { + ResetJitContextTable(jitContext, relationId); + } + } + jitContext = parentContext; + parentContext = jitContext->m_parentContext; + } +} + +extern void InvalidateJitContext(MotJitContext* jitContext, uint64_t relationId, uint8_t invalidFlag /* = 0 */) +{ + // set this context as invalid and all ancestors as child-invalid + MOT_LOG_TRACE("Invalidating %s JIT context %p: %s", + JitContextUsageToString(jitContext->m_usage), + jitContext, + jitContext->m_queryString); + // ATTENTION: all compilation flags are reset here, as we want to attempt to revalidate anyway, but be careful not + // to reset deprecate flag + ResetJitContextCompileState(jitContext); + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + validState |= JIT_CONTEXT_INVALID; + MOT_ATOMIC_STORE(jitContext->m_validState, validState); + + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + // either table or index was dropped. index was already nullified during purge. if this is table drop then we + // should also nullify table pointer. + if (relationId != 0) { + ResetJitContextTable(jitContext, relationId); + } + } + + PropagateValidState(jitContext, invalidFlag, relationId); +} + +static bool ReplenishDatumArray(JitDatumArray* source, JitDatumArray* target, JitContextUsage usage, int depth) +{ + MOT_LOG_TRACE("%d: Replenishing datum array with %u elements", depth, source->m_datumCount); + + if (source->m_datumCount == 0) { + DestroyDatumArray(target, usage); + return true; + } + + if (source->m_datumCount == target->m_datumCount) { + // destroy existing target datum elements and reuse them + for (uint32_t i = 0; i < target->m_datumCount; ++i) { + JitDatum* targetDatum = &target->m_datums[i]; + if (!targetDatum->m_isNull && !IsPrimitiveType(targetDatum->m_type)) { + JitMemFree(DatumGetPointer(targetDatum->m_datum), usage); + } + targetDatum->m_isNull = true; + } + } else { + DestroyDatumArray(target, usage); + uint32_t allocSize = source->m_datumCount * sizeof(JitDatum); + target->m_datums = (JitDatum*)JitMemAlloc(allocSize, usage); + if (target->m_datums == nullptr) { + MOT_LOG_TRACE("Failed allocate %u bytes while replenishing datum array", allocSize); + return false; } - // cleanup constant datum array - DestroyDatumArray(&jitContext->m_constDatums, jitContext->m_usage); + target->m_datumCount = source->m_datumCount; + for (uint32_t i = 0; i < target->m_datumCount; ++i) { + JitDatum* targetDatum = &target->m_datums[i]; + targetDatum->m_isNull = true; + } + } - // cleanup sub-query data array - CleanupJitContextSubQueryDataArray(jitContext); + // clone all datum elements + for (uint32_t i = 0; i < source->m_datumCount; ++i) { + JitDatum* sourceDatum = &source->m_datums[i]; + JitDatum* targetDatum = &target->m_datums[i]; + if (!sourceDatum->m_isNull) { + if (IsPrimitiveType(sourceDatum->m_type)) { + targetDatum->m_datum = sourceDatum->m_datum; + } else { + if (!CloneDatum(sourceDatum->m_datum, sourceDatum->m_type, &targetDatum->m_datum, usage)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "JIT Compile", "Failed to replenish datum array entry"); + DestroyDatumArray(target, usage); + return false; + } + } + } + targetDatum->m_isNull = sourceDatum->m_isNull; + targetDatum->m_type = sourceDatum->m_type; + } + + return true; +} + +static bool ReplenishInvokeParams(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage, int depth) +{ + MOT_LOG_TRACE("%d: Replenishing %u invoke parameters for SP: %s", + depth, + source->m_invokeParamCount, + source->m_invokeContext ? source->m_invokeContext->m_queryString : source->m_invokedQueryString); + if (source->m_invokeParamCount == 0) { + if (target->m_invokeParamCount > 0) { + JitMemFree(target->m_invokeParamInfo, usage); + } + target->m_invokeParamInfo = nullptr; + } else { + size_t allocSize = sizeof(JitParamInfo) * source->m_invokeParamCount; + if (target->m_invokeParamCount < source->m_invokeParamCount) { + void* buf = JitMemRealloc(target->m_invokeParamInfo, allocSize, MOT::MEM_REALLOC_COPY_ZERO, usage); + if (buf == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to reallocate %u bytes for %u invoke parameters", + (unsigned)allocSize, + (unsigned)source->m_invokeParamCount); + return false; + } + target->m_invokeParamInfo = (JitParamInfo*)buf; + } + + MOT_ASSERT(target->m_invokeParamInfo != nullptr); + errno_t erc = memcpy_s(target->m_invokeParamInfo, allocSize, source->m_invokeParamInfo, allocSize); + securec_check(erc, "\0", "\0"); + } + target->m_invokeParamCount = source->m_invokeParamCount; + return true; +} + +static bool ReplenishSubQueryArray(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage, int depth) +{ + MOT_LOG_TRACE( + "%d: Replenishing %u sub-query data items: %s", depth, source->m_subQueryCount, source->m_queryString); + size_t allocSize = sizeof(JitSubQueryContext) * source->m_subQueryCount; + JitMemFree(target->m_subQueryContext, usage); + target->m_subQueryContext = (JitSubQueryContext*)JitMemAllocAligned(allocSize, L1_CACHE_LINE, usage); + if (target->m_subQueryContext == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %u bytes for %u sub-query data array in JIT context object", + (unsigned)allocSize, + (unsigned)source->m_subQueryCount); + return false; + } + target->m_subQueryCount = source->m_subQueryCount; + + for (uint32_t i = 0; i < source->m_subQueryCount; ++i) { + target->m_subQueryContext[i].m_commandType = source->m_subQueryContext[i].m_commandType; + target->m_subQueryContext[i].m_table = source->m_subQueryContext[i].m_table; + target->m_subQueryContext[i].m_tableId = source->m_subQueryContext[i].m_tableId; + target->m_subQueryContext[i].m_index = source->m_subQueryContext[i].m_index; + target->m_subQueryContext[i].m_indexId = source->m_subQueryContext[i].m_indexId; + } + return true; +} + +static bool ReplenishAggregateArray(JitQueryContext* source, JitQueryContext* target, JitContextUsage usage, int depth) +{ + size_t allocSize = sizeof(JitInvokedQueryExecState) * source->m_aggCount; + JitQueryExecState* queryExecState = (JitQueryExecState*)target->m_execState; + if (queryExecState != nullptr) { + void* buf = (JitAggExecState*)JitMemRealloc( + queryExecState->m_aggExecState, allocSize, MOT::MEM_REALLOC_COPY_ZERO, usage); + if (buf == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %u bytes for %u aggregate execution state array in Query JIT context object", + allocSize, + (unsigned)source->m_aggCount); + return false; + } + queryExecState->m_aggExecState = (JitAggExecState*)buf; + } + return true; +} + +static bool ReplenishInvokeContext(JitQueryContext* source, JitQueryContext* target, int depth) +{ + JitContextUsage usage = target->m_usage; + if (target->m_invokedQueryString != nullptr) { + JitMemFree(target->m_invokedQueryString, usage); + target->m_invokedQueryString = nullptr; + } + if (source->m_commandType == JIT_COMMAND_INVOKE) { + if (source->m_invokeContext != nullptr) { + if (target->m_invokeContext == nullptr) { + target->m_invokeContext = (JitFunctionContext*)CloneJitContext(source->m_invokeContext, usage); + if (target->m_invokeContext == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone invocation context"); + return false; + } + } else { + MOT_LOG_TRACE("%d: Replenishing invoked stored procedure context: %s", + depth, + source->m_invokeContext->m_queryString); + if (!ReplenishJitContext(source->m_invokeContext, target->m_invokeContext, depth + 1)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to replenish invocation context"); + return false; + } + } + } else { // unjittable invoked stored procedure + // cleanup previous invoked context + if (target->m_invokeContext != nullptr) { + DestroyJitContext(target->m_invokeContext); + target->m_invokeContext = nullptr; + } + MOT_ASSERT(source->m_invokedQueryString); + target->m_invokedQueryString = DupString(source->m_invokedQueryString, usage); + if (target->m_invokedQueryString == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to clone invoked query string: %s", + source->m_invokedQueryString); + return false; + } + target->m_invokedFunctionOid = source->m_invokedFunctionOid; + target->m_invokedFunctionTxnId = source->m_invokedFunctionTxnId; + } + // in either case replenish parameters passed to invoked stored procedure + if (target->m_invokeContext != nullptr) { + target->m_invokeContext->m_parentContext = target; + } + if (!ReplenishInvokeParams(source, target, usage, depth)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to replenish invocation parameter array"); + return false; + } + } else { + if (target->m_invokeParamCount > 0) { + JitMemFree(target->m_invokeParamInfo, usage); + } + target->m_invokeParamInfo = nullptr; + target->m_invokeParamCount = 0; + } + + MOT_ASSERT((target->m_invokeParamCount == 0 && target->m_invokeParamInfo == nullptr) || + (target->m_invokeParamCount > 0 && target->m_invokeParamInfo != nullptr)); + return true; +} + +static bool ReplenishJitQueryContext(JitQueryContext* source, JitQueryContext* target, int depth) +{ + MOT_LOG_TRACE("%d: Replenishing JIT query context %p into %p (table=%p): %s", + depth, + source, + target, + target->m_table, + source->m_queryString); + + JitQueryContextSetTablesAndIndices(source, target); + target->m_aggCount = source->m_aggCount; + target->m_subQueryCount = source->m_subQueryCount; + + JitContextUsage usage = target->m_usage; + + if (!ReplenishInvokeContext(source, target, depth)) { + MOT_LOG_TRACE("Failed to replenish invoke context"); + return false; + } + + // replenish sub-query tuple descriptor array + if (source->m_subQueryCount > 0) { + if (!ReplenishSubQueryArray(source, target, usage, depth)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to replenish sub-query array"); + return false; + } + } else { + if (target->m_subQueryContext != nullptr) { + JitMemFree(target->m_subQueryContext, usage); + target->m_subQueryContext = nullptr; + } + target->m_subQueryCount = 0; + } + + // replenish aggregate array + if (source->m_aggCount > 0) { + if (!ReplenishAggregateArray(source, target, usage, depth)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to replenish aggregate array"); + return false; + } + } else { + JitExec::JitQueryExecState* targetExecState = (JitExec::JitQueryExecState*)target->m_execState; + if ((targetExecState != nullptr) && (targetExecState->m_aggExecState != nullptr)) { + JitMemFree(targetExecState->m_aggExecState, usage); + targetExecState->m_aggExecState = nullptr; + } + } + + // replenish non-native sort parameters + if (target->m_nonNativeSortParams != nullptr) { + DestroyJitNonNativeSortParams(target->m_nonNativeSortParams, target->m_usage); + target->m_nonNativeSortParams = nullptr; + } + + if (source->m_nonNativeSortParams) { + target->m_nonNativeSortParams = CloneJitNonNativeSortParams(source->m_nonNativeSortParams, usage); + if (target->m_nonNativeSortParams == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to replenish non-native sort params"); + return false; + } + } + + if (target->m_execState != nullptr) { + DestroyQueryExecState(target); + target->m_execState = nullptr; + + if (!AllocJitQueryExecState(target)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to allocate query execution state"); + return false; + } + } + + MOT_LOG_TRACE("%d: Replenished JIT query context %p into %p (table=%p, index=%p): %s", + depth, + source, + target, + target->m_table, + target->m_index, + source->m_queryString); + return true; +} + +static bool ReplenishParamListInfo(JitCallSite* target, JitCallSite* source, JitContextUsage usage, int depth) +{ + if (source->m_queryContext != nullptr) { + MOT_LOG_TRACE("%d: Replenishing param list info of size %d: %s", + depth, + source->m_callParamCount, + source->m_queryContext->m_queryString); + } else { + MOT_LOG_TRACE( + "%d: Replenishing param list info of size %d: %s", depth, source->m_callParamCount, source->m_queryString); + } + if (source->m_callParamCount == 0) { + if (target->m_callParamCount > 0) { + JitMemFree(target->m_callParamInfo, usage); + target->m_callParamInfo = nullptr; + } + } else { + size_t allocSize = sizeof(JitCallParamInfo) * source->m_callParamCount; + if (target->m_callParamCount < source->m_callParamCount) { + void* buf = JitMemRealloc(target->m_callParamInfo, allocSize, MOT::MEM_REALLOC_COPY_ZERO, usage); + if (buf == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to re-allocate %u bytes for sub-query parameter array of size %u in Function JIT context", + allocSize, + source->m_callParamCount); + return false; + } + target->m_callParamInfo = (JitCallParamInfo*)buf; + } + errno_t erc = memcpy_s(target->m_callParamInfo, allocSize, source->m_callParamInfo, allocSize); + securec_check(erc, "\0", "\0"); + } + target->m_callParamCount = source->m_callParamCount; + return true; +} + +static bool ResizeSubQueryArray(JitFunctionContext* source, JitFunctionContext* target) +{ + JitContextUsage usage = target->m_usage; + JitFunctionExecState* functionExecState = (JitFunctionExecState*)target->m_execState; + uint32_t origSubQueryCount = target->m_SPSubQueryCount; + uint64_t allocSize = sizeof(JitCallSite) * source->m_SPSubQueryCount; + + // resize sub-query array if needed (and the execution state array) + if (target->m_SPSubQueryCount < source->m_SPSubQueryCount) { + void* buf = JitMemRealloc(target->m_SPSubQueryList, allocSize, MOT::MEM_REALLOC_COPY_ZERO, usage); + if (buf == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %lu bytes for %u sub-query data array in Function JIT context object", + allocSize, + target->m_SPSubQueryCount); + return false; + } + target->m_SPSubQueryList = (JitCallSite*)buf; + target->m_SPSubQueryCount = source->m_SPSubQueryCount; + + if (functionExecState != nullptr) { + // the execution state must be totally destroyed, as the sub-queries might have totally changed + MOT_LOG_TRACE("Destroying all execution state objects (target query count is too small)"); + for (uint32_t i = 0; i < origSubQueryCount; ++i) { + JitInvokedQueryExecState* subExecState = &functionExecState->m_invokedQueryExecState[i]; + DestroyJitInvokeExecState(subExecState); + } + // now reallocate the entire array and zero it, so PrepareCallSite will create all missing members + allocSize = sizeof(JitInvokedQueryExecState) * source->m_SPSubQueryCount; + MOT_LOG_TRACE( + "Allocating %lu bytes for %u invoked query exec state objects", allocSize, source->m_SPSubQueryCount); + buf = JitMemRealloc(functionExecState->m_invokedQueryExecState, allocSize, MOT::MEM_REALLOC_ZERO, usage); + if (buf == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %lu bytes for %u sub-query exec state array in Function JIT context object", + allocSize, + target->m_SPSubQueryCount); + return false; + } + functionExecState->m_invokedQueryExecState = (JitInvokedQueryExecState*)buf; + MOT_LOG_TRACE("Invoke exec state array re-allocated at: %p (%lu bytes)", buf, allocSize); + } + } else { + // make sure then entire execution state is discarded and rebuilt from scratch + if (functionExecState != nullptr) { + // the execution state must be totally destroyed, as the sub-queries might have totally changed + MOT_LOG_TRACE("Destroying all execution state objects (target query count is large enough)"); + for (uint32_t i = 0; i < origSubQueryCount; ++i) { + JitInvokedQueryExecState* subExecState = &functionExecState->m_invokedQueryExecState[i]; + MOT_LOG_TRACE("Destroying sub-query %u invoke exec state at: %p", i, subExecState); + DestroyJitInvokeExecState(subExecState); + } + } + } + + return true; +} + +static bool ReplenishExistingSubQueries( + uint32_t replenishCount, JitFunctionContext* source, JitFunctionContext* target, int depth) +{ + MOT_LOG_TRACE("Replenishing (%u) existing sub-queries for JIT function context %p", replenishCount, target); + + JitContextUsage usage = target->m_usage; + for (uint32_t i = 0; i < replenishCount; ++i) { + JitCallSite* sourceCallSite = &source->m_SPSubQueryList[i]; + JitCallSite* targetCallSite = &target->m_SPSubQueryList[i]; + if (sourceCallSite->m_queryContext) { + MOT_LOG_TRACE("%d: Replenishing from jittable sub-query %u: %s", + depth, + i, + sourceCallSite->m_queryContext->m_queryString); + } else { + MOT_LOG_TRACE( + "%d: Replenishing from non-jittable sub-query %u: %s", depth, i, sourceCallSite->m_queryString); + } + if (targetCallSite->m_queryString != nullptr) { + JitMemFree(targetCallSite->m_queryString, target->m_usage); + targetCallSite->m_queryString = nullptr; + } + if (sourceCallSite->m_queryContext == nullptr) { + if (targetCallSite->m_queryContext != nullptr) { + DestroyJitContext(targetCallSite->m_queryContext); + targetCallSite->m_queryContext = nullptr; + } + targetCallSite->m_queryString = DupString(sourceCallSite->m_queryString, target->m_usage); + if (targetCallSite->m_queryString == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to clone call site query string: %s", + sourceCallSite->m_queryString); + return false; + } + MOT_LOG_TRACE("ReplenishJitFunctionContext(): Cloned string %p on %s scope into call site %p: %s", + targetCallSite->m_queryString, + IsJitContextUsageGlobal(target->m_usage) ? "global" : "local", + targetCallSite, + targetCallSite->m_queryString); + } else { + if (targetCallSite->m_queryContext == nullptr) { + // replenish from jittable sub-query into previously non-jittable sub-query + targetCallSite->m_queryContext = CloneJitContext(sourceCallSite->m_queryContext, usage); + if (targetCallSite->m_queryContext == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to replenish sub-query context %d for function JIT context (clone failed)", + i); + return false; + } + } else if (!ReplenishJitContext( + sourceCallSite->m_queryContext, targetCallSite->m_queryContext, depth + 1)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to replenish sub-query context %d for function JIT context", + i); + return false; + } + targetCallSite->m_queryContext->m_parentContext = target; + } + targetCallSite->m_queryCmdType = sourceCallSite->m_queryCmdType; + if (!ReplenishParamListInfo(targetCallSite, sourceCallSite, usage, depth)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to replenish sub-query %d parameters for function JIT context", + sourceCallSite->m_callParamCount); + return false; + } + + if (targetCallSite->m_tupDesc != nullptr) { + FreeTupleDesc(targetCallSite->m_tupDesc); + targetCallSite->m_tupDesc = nullptr; + } + if (sourceCallSite->m_tupDesc != nullptr) { + if (!CloneTupleDesc(sourceCallSite->m_tupDesc, &targetCallSite->m_tupDesc, usage)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to clone sub-query %d tuple descriptor for function JIT context", + i); + return false; + } + } + targetCallSite->m_exprIndex = sourceCallSite->m_exprIndex; + targetCallSite->m_isUnjittableInvoke = sourceCallSite->m_isUnjittableInvoke; + targetCallSite->m_isModStmt = sourceCallSite->m_isModStmt; + targetCallSite->m_isInto = sourceCallSite->m_isInto; + } + + return true; +} +/* function name:ReplenishResultDescriptor + function purpose:Replenishes or updates the result descriptors of a target JIT function context + using the source JIT function context. It first frees any existing result descriptors + in the target context before cloning from the source. + input:source The source JIT function context from which result descriptors are to be cloned.target The target JIT function context which will be replenished with result descriptors. + output: Returns true if replenishing was successful, otherwise returns false. + note:none + annotator:liushifa + annotate time:2023/10/05 22:19:54 + contact:3325287047@qq.com +*/ +static bool ReplenishResultDescriptor(JitFunctionContext* source, JitFunctionContext* target) +{ + // If target already has result descriptors, free them first to avoid memory leaks + if (target->m_resultTupDesc != nullptr) { + FreeTupleDesc(target->m_resultTupDesc); + target->m_resultTupDesc = nullptr; + } + if (target->m_rowTupDesc != nullptr) { + FreeTupleDesc(target->m_rowTupDesc); + target->m_rowTupDesc = nullptr; + } + // Clone the result descriptors from source to target + if (!CloneResultDescriptors(source, target)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone result descriptors for function JIT context"); + return false; + } + // Return true if the replenishing was successful + return true; +} + +static bool ReplenishParamTypes(JitFunctionContext* source, JitFunctionContext* target) +{ + JitContextUsage usage = target->m_usage; + if (target->m_paramTypes != nullptr) { + JitMemFree(target->m_paramTypes, usage); + target->m_paramTypes = nullptr; + } + + target->m_paramCount = source->m_paramCount; + if (source->m_paramCount > 0) { + uint64_t allocSize = sizeof(Oid) * target->m_paramCount; + target->m_paramTypes = (Oid*)JitMemAlloc(allocSize, usage); + if (target->m_paramTypes == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Generate JIT Code", + "Failed to allocate %u bytes for %u sub-query parameter array in Function JIT context object", + allocSize, + target->m_paramCount); + return false; + } + errno_t erc = memcpy_s(target->m_paramTypes, allocSize, source->m_paramTypes, allocSize); + securec_check(erc, "\0", "\0"); + } + + return true; +} + +static bool ReplenishJitFunctionContext(JitFunctionContext* source, JitFunctionContext* target, int depth) +{ + MOT_LOG_TRACE( + "%d: Replenishing JIT function context %p using %p: %s", depth, target, source, source->m_queryString); + JitContextUsage usage = target->m_usage; + target->m_functionOid = source->m_functionOid; + MOT_LOG_TRACE( + "Replenishing function txn id from %" PRIu64 " to %" PRIu64, target->m_functionTxnId, source->m_functionTxnId); + target->m_functionTxnId = source->m_functionTxnId; + target->m_SPArgCount = source->m_SPArgCount; + + // NOTE: in any case of failure caller will call destroy function for safe cleanup + + JitFunctionExecState* functionExecState = (JitFunctionExecState*)target->m_execState; + + // ATTENTION: Cache the original sub query count in the local variable, because target->m_SPSubQueryCount will be + // modified in ResizeSubQueryArray(). + uint32_t origSubQueryCount = target->m_SPSubQueryCount; + + // resize sub-query array if needed (and the execution state array) + if (!ResizeSubQueryArray(source, target)) { + MOT_LOG_TRACE( + "ReplenishJitFunctionContext(): Failed to resize sub-query array for JIT function context %p", target); + return false; + } + + // release compiled function + if ((functionExecState != nullptr) && (functionExecState->m_function != nullptr)) { + MOT_ASSERT(functionExecState->m_function->use_count > 0); + --functionExecState->m_function->use_count; + MOT_LOG_TRACE("ReplenishJitFunctionContext(): Decreased use count of function %p to %lu: %s", + functionExecState->m_function, + functionExecState->m_function->use_count, + target->m_queryString); + functionExecState->m_function = nullptr; + } + + // replenish existing sub-queries + uint32_t replenishCount = std::min(origSubQueryCount, source->m_SPSubQueryCount); + MOT_LOG_TRACE("Replenish count is: %u", replenishCount); + if (!ReplenishExistingSubQueries(replenishCount, source, target, depth)) { + MOT_LOG_TRACE("ReplenishJitFunctionContext(): Failed to replenish existing sub-queries (%u) for JIT function " + "context %p", + replenishCount, + target); + return false; + } + + // clone newly added sub-queries + for (uint32_t i = replenishCount; i < source->m_SPSubQueryCount; ++i) { + JitCallSite* sourceCallSite = &source->m_SPSubQueryList[i]; + JitCallSite* targetCallSite = &target->m_SPSubQueryList[i]; + if (sourceCallSite->m_queryContext) { + MOT_LOG_TRACE("%d: Cloning new sub-query %u: %s", depth, i, sourceCallSite->m_queryContext->m_queryString); + } else { + MOT_LOG_TRACE("%d: Cloning new sub-query %u: %s", depth, i, sourceCallSite->m_queryString); + } + if (!CloneCallSite(target, i, sourceCallSite, targetCallSite, usage)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "Generate JIT Code", "Failed to clone call-site %d for function JIT context", i); + return false; + } + if (targetCallSite->m_queryContext != nullptr) { + targetCallSite->m_queryContext->m_parentContext = target; + } + } + + // release resource of unused sub-queries and their execution state objects + if (origSubQueryCount > source->m_SPSubQueryCount) { + for (uint32_t i = source->m_SPSubQueryCount; i < origSubQueryCount; ++i) { + JitCallSite* callSite = &target->m_SPSubQueryList[i]; + if (callSite->m_queryContext) { + MOT_LOG_TRACE( + "%d: Removing unused sub-query %u: %s", depth, i, callSite->m_queryContext->m_queryString); + } else { + MOT_LOG_TRACE("%d: Removing unused sub-query %u: %s", depth, i, callSite->m_queryString); + } + DestroyCallSite(callSite, target->m_usage); + + if ((functionExecState != nullptr) && (functionExecState->m_invokedQueryExecState != nullptr)) { + DestroyJitInvokeExecState(&functionExecState->m_invokedQueryExecState[i]); + } + } + } + target->m_SPSubQueryCount = source->m_SPSubQueryCount; + + // replenish result tuple descriptor (even though highly unexpected) + if (!ReplenishResultDescriptor(source, target)) { + MOT_LOG_TRACE( + "ReplenishJitFunctionContext(): Failed to clone result tuple desc for function JIT context %p", target); + return false; + } + + // replenish parameter types array + if (!ReplenishParamTypes(source, target)) { + MOT_LOG_TRACE("ReplenishJitFunctionContext(): Failed to replenish parameter types array for function JIT " + "context %p", + target); + return false; + } + + if (target->m_execState != nullptr) { + DestroyFunctionExecState(target); + target->m_execState = nullptr; + + if (!AllocJitFunctionExecState(target)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Generate JIT Code", "Failed to allocate function execution state"); + return false; + } + } + + MOT_LOG_TRACE("%d: Replenished JIT function context %p into %p: %s", depth, source, target, source->m_queryString); + return true; +} + +static bool ReplenishJitContext(MotJitContext* source, MotJitContext* target, int depth) +{ + if (source->m_contextType != target->m_contextType) { + MOT_REPORT_ERROR(MOT_ERROR_INVALID_ARG, + "JIT Compile", + "Cannot replenish %s JIT context %p from %s JIT context %p: mismatching context type", + JitContextTypeToString(target->m_contextType), + target, + JitContextTypeToString(source->m_contextType), + source); + return false; + } + const char* itemName = (source->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) ? "query" : "function"; + MOT_LOG_TRACE( + "%d: Replenishing JIT context %p from %p of %s: %s", depth, target, source, itemName, source->m_queryString); + + // no need to clone module/codegen object (they are safe in the source context) + JitContextUsage usage = target->m_usage; + target->m_llvmFunction = source->m_llvmFunction; + target->m_llvmSPFunction = source->m_llvmSPFunction; + target->m_commandType = source->m_commandType; + target->m_validState = source->m_validState; + target->m_queryString = source->m_queryString; + if (!ReplenishDatumArray(&source->m_constDatums, &target->m_constDatums, usage, depth)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "JIT Compile", "Failed to clone constant datum array"); + return false; + } + + // pay attention: the JIT source/context might change! + if ((target->m_jitSource != source->m_jitSource) || (target->m_sourceJitContext != source)) { + MOT_LOG_TRACE("%d: Replenish target %p/%p from source %p/%p", + depth, + target->m_jitSource, + target->m_sourceJitContext, + source->m_jitSource, + source); + + // attention: even if source did not change we remove and add it again so that removal from deprecate source + // context takes place properly + PurgeJitContext(target, 0); // full cleanup before detaching from source + RemoveJitSourceContext(target->m_jitSource, target); + AddJitSourceContext(source->m_jitSource, target); + } + + bool result = true; + if (source->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + result = ReplenishJitQueryContext((JitQueryContext*)source, (JitQueryContext*)target, depth); + } else if (source->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + result = ReplenishJitFunctionContext((JitFunctionContext*)source, (JitFunctionContext*)target, depth); + } + + if (result) { + MOT_LOG_TRACE("%d: Replenished successfully JIT context %p from %p of %s: %s", + depth, + target, + source, + itemName, + source->m_queryString); + MOT_ATOMIC_STORE(target->m_validState, JIT_CONTEXT_VALID); + } else { + MOT_LOG_TRACE( + "Failed to replenish JIT context %p from %p of %s: %s", target, source, itemName, source->m_queryString); + } + + return result; +} + +static bool RevalidateJitQueryContext(MotJitContext* jitContext) +{ + MOT_LOG_TRACE("Re-validating JIT query context %p of query: %s", jitContext, jitContext->m_queryString); + if (jitContext->m_commandType != JIT_COMMAND_INVOKE) { + // this is a simple query, we declare revalidation fails to force destruction and re-creation + MOT_LOG_TRACE("Skipping revalidation of simple query to force code regeneration"); + return false; + } + + // we first make sure that the invoked function is valid (this is a secondary global context) + JitQueryContext* queryContext = (JitQueryContext*)jitContext; + MotJitContext* invokedContext = queryContext->m_invokeContext; + if (invokedContext == nullptr) { + MOT_LOG_TRACE("Skipping revalidation of invoke query into unjittable SP: %s", jitContext->m_queryString); + return true; // revalidation is OK + } + uint8_t validState = MOT_ATOMIC_LOAD(invokedContext->m_validState); + MOT_ASSERT(invokedContext->m_usage == JIT_CONTEXT_GLOBAL_SECONDARY); + MOT_ASSERT(invokedContext->m_isSourceContext == 0); + MOT_LOG_TRACE("Triggering revalidation of invoked context %p", invokedContext); + if (!RevalidateJitContext(invokedContext)) { + MOT_LOG_TRACE("Invoked context %p revalidation failed", invokedContext); + return false; + } + + // now if the function signature changed (e.g. default values changed), we need to re-generate code for the invoked + // query, we do so by dropping the query altogether and forcing code generation from scratch + if (validState & JIT_CONTEXT_INVALID) { // SP itself was replaced or dropped and recreated + MOT_LOG_TRACE( + "Forcing code regeneration from scratch for INVOKE of re-created SP: %s", jitContext->m_queryString); + return false; + } + + // otherwise the SP context was already replenished + MOT_ATOMIC_STORE(jitContext->m_validState, JIT_CONTEXT_VALID); + MOT_LOG_TRACE("Re-validated JIT query context %p of query: %s", jitContext, jitContext->m_queryString); + return true; +} + +static bool RevalidateJitFunctionContext(MotJitContext* jitContext) +{ + MOT_LOG_TRACE("Re-validating JIT function %p of function: %s", jitContext, jitContext->m_queryString); + // we arrive here when a sub-query or sub-SP was invalidated + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + if (validState & (JIT_CONTEXT_CHILD_QUERY_INVALID | JIT_CONTEXT_CHILD_SP_INVALID)) { + // now regenerate sub-queries and sub-SPs (sub-queries and sub-SPs are treated alike) + // we expect them to be cloned from the already regenerated source + MOT_LOG_TRACE("Regenerating code for SP sub-queries"); + if (!JitReCodegenFunctionQueries(jitContext)) { + MOT_LOG_TRACE("Failed to revalidate JIT SP sub-queries: %s", jitContext->m_queryString); + return false; + } + } + + MOT_LOG_TRACE("Re-validated JIT function context %p of query: %s", jitContext, jitContext->m_queryString); + return true; +} + +static bool ReattachDeprecateContext(MotJitContext* jitContext, TransactionId functionTxnId, const char*& queryString) +{ + // first check for deprecate source + uint8_t validState = MOT_ATOMIC_LOAD(jitContext->m_validState); + if (validState & JIT_CONTEXT_DEPRECATE) { + // find the latest matching source + MOT_LOG_TRACE("Revalidating deprecate source"); + bool shouldPopNamespace = false; + if ((jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) && + IsJitSubContextInline(jitContext) && + (jitContext->m_parentContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION)) { + // ATTENTION: this use case is ONLY when executing a jitted SP query, during lock-acquire via + // RevalidateCachedQuery() (see JitExecSubQuery() in jit_helpers.cpp) + JitFunctionContext* parentContext = (JitFunctionContext*)jitContext->m_parentContext; + // Try to get the function name to see if the procedure still exists. + char* funcName = get_func_name(parentContext->m_functionOid); + if (funcName == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_CONCURRENT_MODIFICATION, + "JIT Revalidate", + "Cannot attach to new source after deprecate: function %u concurrently dropped", + parentContext->m_functionOid); + // ATTENTION: we remain in deprecate state - the JIT SP query will be executed as non-jittable as + // many times as needed, until the calling SP execution finishes, and then on next round gets + // revalidated properly with this deprecate sub-query (see JitReCodegenFunctionQueries()) + return false; + } + if (!PushJitSourceNamespace(parentContext->m_functionOid, parentContext->m_queryString)) { + MOT_LOG_TRACE("JIT Revalidate: Failed to push JIT source namespace for stored procedure %s (%u)", + funcName, + parentContext->m_functionOid); + pfree_ext(funcName); + return false; + } + pfree_ext(funcName); + shouldPopNamespace = true; + } + + // NOTE: Once we remove the session-local JIT context from the JIT source, it is possible that the source + // be deprecated, freed and reused for completely different query. So removing the context from old source + // and adding it to the new source should be done atomically within the source map lock. Otherwise, it will + // result in a complete mess (JIT context getting attached to a completely wrong source). + LockJitSourceMap(); + JitSource* newSource = GetCachedJitSource(queryString); + if (shouldPopNamespace) { + PopJitSourceNamespace(); + } + if (newSource == nullptr) { + MOT_LOG_TRACE("Failed to find valid source"); + UnlockJitSourceMap(); + MarkJitContextErrorCompile(jitContext); // make sure we are in error state + return false; + } + + if (!IsSimpleQueryContext(jitContext) && IsPrematureRevalidation(newSource, functionTxnId)) { + MOT_LOG_TRACE("Skipping premature re-validation of JIT context %p with source %p, new source %p: %s", + jitContext, + jitContext->m_jitSource, + newSource, + queryString); + UnlockJitSourceMap(); + MarkJitContextErrorCompile(jitContext); // make sure we are in error state + return false; + } + + PurgeJitContext(jitContext, 0); // full cleanup before detaching from source + RemoveJitSourceContext(jitContext->m_jitSource, jitContext); + AddJitSourceContext(newSource, jitContext); + queryString = jitContext->m_jitSource->m_queryString; // get the query string again from new source + MOT_ATOMIC_STORE(jitContext->m_validState, GetJitSourceValidState(newSource)); + UnlockJitSourceMap(); + MOT_LOG_TRACE("Valid state updated to: %x", jitContext->m_validState); + } + + return true; +} + +static bool CleanupJitContextExecState(MotJitContext* jitContext) +{ + if (jitContext->m_execState != nullptr) { + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + DestroyQueryExecState((JitQueryContext*)jitContext); + jitContext->m_execState = nullptr; + + if (!AllocJitQueryExecState((JitQueryContext*)jitContext)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Jit Revalidate", "Failed to allocate query execution state"); + return false; + } + } else { + DestroyFunctionExecState((JitFunctionContext*)jitContext); + jitContext->m_execState = nullptr; + + if (!AllocJitFunctionExecState((JitFunctionContext*)jitContext)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, "Jit Revalidate", "Failed to allocate function execution state"); + return false; + } + } + } + + return true; +} + +extern bool RevalidateJitContext(MotJitContext* jitContext, TransactionId functionTxnId /* = InvalidTransactionId */) +{ + const char* queryString = jitContext->m_jitSource->m_queryString; + MOT_LOG_TRACE("Re-validating JIT context %p of query: %s", jitContext, queryString); + if (!jitContext->m_isSourceContext) { + // first check for deprecate source + // NOTE: queryString is passed by reference. If the deprecated context is getting attached to a new source, + // it will be updated accordingly. + if (!ReattachDeprecateContext(jitContext, functionTxnId, queryString)) { + MOT_LOG_TRACE("Failed to revalidate JIT context %p, couldn't attach to a new JIT source", jitContext); + return false; + } + + // Cleanup execution state before revalidation. It is possible that JIT context is not purged during create + // index as other DML operations can be executed concurrently during create index. So we must cleanup the old + // execution state before revalidation. Otherwise, we might end up with old keys/iterators in execution state, + // but new index pointers in JIT context. + if (!CleanupJitContextExecState(jitContext)) { + MOT_LOG_TRACE("Failed to revalidate JIT context %p, couldn't cleanup execution state", jitContext); + return false; + } + + // ATTENTION: At this point it is possible that some other session is concurrently compiling the same query. In + // this case, we don't really care, we just try to revalidate, but we only make sure not to overwrite "done" or + // "error" compile state with "pending" (so the race is resolved by flag precedence). + JitCodegenState codegenState = RevalidateJitSourceTxn(jitContext->m_jitSource, functionTxnId); + if (codegenState != JitCodegenState::JIT_CODEGEN_READY) { + bool pendingCompile = ((codegenState == JitCodegenState::JIT_CODEGEN_UNAVAILABLE) || + (codegenState == JitCodegenState::JIT_CODEGEN_PENDING)); + if (pendingCompile) { + MOT_LOG_TRACE("Revalidate JIT context %p pending compilation with state %s: %s", + jitContext, + JitCodegenStateToString(codegenState), + queryString); + // the following call will not set pending state in case done/error was set (if compilation finished + // after we failed to revalidate, then we will catch that on the next invocation) + MarkJitContextPendingCompile(jitContext); + } else { + MOT_LOG_TRACE("Failed to revalidate JIT context %p, source re-validation failed with state %s: %s", + jitContext, + JitCodegenStateToString(codegenState), + queryString); + // we remain in code-gen error state, until some DDL comes and raises invalid flag, which will trigger + // another attempt to revalidate. We just make sure revalidate is not re-attempted by setting the valid + // state to compile-error (and also for making sure that OpFusion does not use JIT) + MarkJitContextErrorCompile(jitContext); + } + return false; + } + + // revalidate succeeded, so we clear all compilation flags + // we also want to avoid another attempt to revalidate - so we just set valid state to zero + // we hold query locks so there is no possible race with concurrent compilation after successful revalidate + MOT_LOG_TRACE("Revalidate JIT context %p source succeeded, now replenishing: %s", jitContext, queryString); + MOT_ATOMIC_STORE(jitContext->m_validState, JIT_CONTEXT_VALID); + + // although we should be protected by plan locks, we still prefer to guard access to source context + if (jitContext->m_jitSource->m_usage == JIT_CONTEXT_GLOBAL) { + LockJitSource(jitContext->m_jitSource); + } + if (jitContext->m_jitSource->m_sourceJitContext == nullptr) { + MOT_LOG_TRACE("Failed to revalidate JIT context %p: concurrent update", jitContext); + if (jitContext->m_jitSource->m_usage == JIT_CONTEXT_GLOBAL) { + UnlockJitSource(jitContext->m_jitSource); + } + MarkJitContextErrorCompile(jitContext); + return false; + } + if (!ReplenishJitContext(jitContext->m_jitSource->m_sourceJitContext, jitContext, 0)) { + MOT_LOG_TRACE("Failed to revalidate JIT context %p: replenish from source failed", jitContext); + if (jitContext->m_jitSource->m_usage == JIT_CONTEXT_GLOBAL) { + UnlockJitSource(jitContext->m_jitSource); + } + MarkJitContextErrorCompile(jitContext); + return false; + } + if (jitContext->m_jitSource->m_usage == JIT_CONTEXT_GLOBAL) { + UnlockJitSource(jitContext->m_jitSource); + } + + // we force prepare after replenish on top level session-local context (we do not prepare for execution + // secondary global context objects in JIT source objects) + if ((jitContext->m_usage == JIT_CONTEXT_LOCAL) && (jitContext->m_execState != nullptr)) { + MOT_ATOMIC_STORE(jitContext->m_execState->m_purged, true); + } + } else { + // revalidate source context + MOT_LOG_TRACE("Re-validating JIT source context %p of query: %s", jitContext, queryString); + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + if (!RevalidateJitQueryContext(jitContext)) { + MOT_LOG_TRACE("Failed to revalidate JIT query source context %p of query: %s", jitContext, queryString); + return false; + } + } else { + if (!RevalidateJitFunctionContext(jitContext)) { + MOT_LOG_TRACE( + "Failed to revalidate JIT function source context %p of function: %s", jitContext, queryString); + return false; + } + } + } + + MOT_ATOMIC_STORE(jitContext->m_validState, JIT_CONTEXT_VALID); + MOT_LOG_TRACE("Re-validated JIT context %p of query: %s", jitContext, queryString); + return true; +} + +extern const char* JitContextValidStateToString(uint8_t validState) +{ + // we disregard relation-invalid and deprecate bits + validState = validState & ~JIT_CONTEXT_RELATION_INVALID; + validState = validState & ~JIT_CONTEXT_DEPRECATE; + if (validState == JIT_CONTEXT_VALID) { + return "valid"; + } + if (validState == JIT_CONTEXT_INVALID) { + return "invalid"; + } + if (validState == JIT_CONTEXT_CHILD_QUERY_INVALID) { + return "sub-query invalid"; + } + if (validState == JIT_CONTEXT_CHILD_SP_INVALID) { + return "sub-SP invalid"; + } + if (validState == (JIT_CONTEXT_CHILD_QUERY_INVALID | JIT_CONTEXT_CHILD_SP_INVALID)) { + return "sub-query/SP invalid"; + } + if (validState == JIT_CONTEXT_PENDING_COMPILE) { + return "pending compile"; + } + if (validState == JIT_CONTEXT_DONE_COMPILE) { + return "done compile"; + } + if (validState == JIT_CONTEXT_ERROR_COMPILE) { + return "compile error"; + } + return "N/A"; +} + +static inline bool RefersRelation(JitQueryContext* jitContext, uint64_t relationId) +{ + if (((jitContext->m_table != nullptr) && (jitContext->m_tableId == relationId)) || + ((jitContext->m_index != nullptr) && (jitContext->m_indexId == relationId)) || + ((jitContext->m_innerTable != nullptr) && (jitContext->m_innerTableId == relationId)) || + ((jitContext->m_innerIndex != nullptr) && (jitContext->m_innerIndexId == relationId))) { + return true; + } + return false; +} + +static inline bool RefersRelation(JitSubQueryContext* subQueryContext, uint64_t relationId) +{ + if (((subQueryContext->m_table != nullptr) && (subQueryContext->m_tableId == relationId)) || + ((subQueryContext->m_index != nullptr) && (subQueryContext->m_indexId == relationId))) { + return true; + } + return false; +} + +static void PurgeJitQueryContext(JitQueryContext* jitContext, uint64_t relationId) +{ + bool refersRelation = false; + + if ((relationId == 0) || RefersRelation(jitContext, relationId)) { + refersRelation = true; + } + + if (!refersRelation) { + for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { + JitSubQueryContext* subQueryContext = &jitContext->m_subQueryContext[i]; + if (RefersRelation(subQueryContext, relationId)) { + refersRelation = true; + break; + } + } + } + + if (refersRelation) { + // If the relation is referred either in primary or inner or sub-query context, purge everything as the + // containing jit-source is marked as expired. + JitQueryExecState* execState = (JitQueryExecState*)jitContext->m_execState; // cleanup keys(s) + MOT_LOG_TRACE("Purging JIT context %p primary keys by relation id %" PRIu64, jitContext, relationId); CleanupJitContextPrimary(jitContext); // cleanup JOIN keys(s) + MOT_LOG_TRACE("Purging JIT context %p inner keys by relation id %" PRIu64, jitContext, relationId); CleanupJitContextInner(jitContext); - // cleanup bitmap set - if (jitContext->m_bitmapSet != nullptr) { - MOT::MemSessionFree(jitContext->m_bitmapSet->GetData()); - jitContext->m_bitmapSet->MOT::BitmapSet::~BitmapSet(); - MOT::MemSessionFree(jitContext->m_bitmapSet); - jitContext->m_bitmapSet = nullptr; - } - - // cleanup code generator (only in global-usage) - if (jitContext->m_codeGen && (jitContext->m_usage == JIT_CONTEXT_GLOBAL)) { - FreeGsCodeGen(jitContext->m_codeGen); - jitContext->m_codeGen = nullptr; - } - - // cleanup null argument array - if (jitContext->m_argIsNull != nullptr) { - MOT::MemSessionFree(jitContext->m_argIsNull); - jitContext->m_argIsNull = nullptr; - } - - // cleanup TVM function (only in global-usage) - if (jitContext->m_tvmFunction && (jitContext->m_usage == JIT_CONTEXT_GLOBAL)) { - delete jitContext->m_tvmFunction; - jitContext->m_tvmFunction = nullptr; - } - - // cleanup TVM execution context - if (jitContext->m_execContext != nullptr) { - tvm::freeExecContext(jitContext->m_execContext); - jitContext->m_execContext = nullptr; - } - - FreeJitContext(jitContext); - } -} - -extern void PurgeJitContext(JitContext* jitContext, uint64_t relationId) -{ - if (jitContext != nullptr) { -#ifdef MOT_JIT_DEBUG - MOT_LOG_TRACE("Purging JIT context %p by external table %" PRIu64 " with %" PRIu64 " executions of query: %s", - jitContext, - relationId, - jitContext->m_execCount, - jitContext->m_queryString); -#else - MOT_LOG_TRACE("Purging %s JIT context %p by external table %" PRIu64 " of query: %s", - jitContext->m_usage == JIT_CONTEXT_GLOBAL ? "global" : "session-local", - jitContext, - relationId, - jitContext->m_queryString); -#endif - - // cleanup keys(s) - if ((jitContext->m_table != nullptr) && (jitContext->m_table->GetTableExId() == relationId)) { - MOT_LOG_TRACE("Purging JIT context %p primary keys by relation id %" PRIu64, jitContext, relationId); - CleanupJitContextPrimary(jitContext); - } - - // cleanup JOIN keys(s) - if ((jitContext->m_innerTable != nullptr) && (jitContext->m_innerTable->GetTableExId() == relationId)) { - MOT_LOG_TRACE("Purging JIT context %p inner keys by relation id %" PRIu64, jitContext, relationId); - CleanupJitContextInner(jitContext); - } - // cleanup sub-query keys for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { - JitContext::SubQueryData* subQueryData = &jitContext->m_subQueryData[i]; - if ((subQueryData->m_table != nullptr) && (subQueryData->m_table->GetTableExId() == relationId)) { - MOT_LOG_TRACE( - "Purging sub-query %u data in JIT context %p by relation id %" PRIu64, i, jitContext, relationId); - CleanupJitContextSubQueryData(subQueryData); - } + JitSubQueryContext* subQueryContext = &jitContext->m_subQueryContext[i]; + MOT_LOG_TRACE( + "Purging sub-query %u data in JIT context %p by relation id %" PRIu64, i, jitContext, relationId); + CleanupJitSubQueryContext(subQueryContext, &execState->m_subQueryExecState[i]); + } + } + + // cleanup function context for INVOKE query + if (jitContext->m_commandType == JIT_COMMAND_INVOKE) { + MOT_LOG_TRACE("Purging sub-function of JIT INVOKE context %p by relationId %" PRIu64, jitContext, relationId); + PurgeJitContext(jitContext->m_invokeContext, relationId); + } +} + +static void PurgeJitFunctionContext(JitFunctionContext* jitContext, uint64_t relationId) +{ + // purge JIT context in each sub-query call site + for (uint32_t i = 0; i < jitContext->m_SPSubQueryCount; ++i) { + JitCallSite* callSite = &jitContext->m_SPSubQueryList[i]; + if (callSite->m_queryContext != nullptr) { + PurgeJitContext(callSite->m_queryContext, relationId); } } } -static void CleanupJitContextPrimary(JitContext* jitContext) +static void CleanupJitContextPrimary(JitQueryContext* jitContext, bool isDropCachedPlan /* = false */) { - if (jitContext->m_table) { - if (jitContext->m_index) { - if (jitContext->m_searchKey) { - jitContext->m_index->DestroyKey(jitContext->m_searchKey); - jitContext->m_searchKey = nullptr; - } + CleanupExecStatePrimary(jitContext, (JitQueryExecState*)jitContext->m_execState, isDropCachedPlan); + jitContext->m_index = nullptr; + jitContext->m_table = nullptr; +} - if (jitContext->m_endIteratorKey != nullptr) { - jitContext->m_index->DestroyKey(jitContext->m_endIteratorKey); - jitContext->m_endIteratorKey = nullptr; - } +static void CleanupJitContextInner(JitQueryContext* jitContext, bool isDropCachedPlan /* = false */) +{ + CleanupExecStateInner(jitContext, (JitQueryExecState*)jitContext->m_execState, isDropCachedPlan); + jitContext->m_innerIndex = nullptr; + jitContext->m_innerTable = nullptr; +} - if (jitContext->m_beginIterator != nullptr) { - destroyIterator(jitContext->m_beginIterator); - jitContext->m_beginIterator = nullptr; +static void CleanupJitSubQueryContextArray(JitQueryContext* jitContext, bool isDropCachedPlan /* = false */) +{ + if (jitContext->m_subQueryContext != nullptr) { + if (jitContext->m_execState != nullptr) { + JitQueryExecState* execState = (JitQueryExecState*)jitContext->m_execState; + MOT_LOG_TRACE("Cleaning up sub-query data array in JIT context %p", jitContext); + if (execState->m_subQueryExecState != nullptr) { + for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { + JitSubQueryContext* subQueryContext = &jitContext->m_subQueryContext[i]; + MOT_LOG_TRACE("Cleaning up sub-query %u data in JIT context %p", i, jitContext); + CleanupJitSubQueryContext(subQueryContext, &execState->m_subQueryExecState[i]); + } + } else { + for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { + JitSubQueryContext* subQueryContext = &jitContext->m_subQueryContext[i]; + MOT_LOG_TRACE("Cleaning up sub-query %u data in JIT context %p", i, jitContext); + subQueryContext->m_index = nullptr; + subQueryContext->m_table = nullptr; + } } - - if (jitContext->m_endIterator != nullptr) { - destroyIterator(jitContext->m_endIterator); - jitContext->m_endIterator = nullptr; - } - - jitContext->m_index = nullptr; - } - - // cleanup JOIN outer row copy - if (jitContext->m_outerRowCopy != nullptr) { - jitContext->m_table->DestroyRow(jitContext->m_outerRowCopy); - jitContext->m_outerRowCopy = nullptr; } + JitMemFree(jitContext->m_subQueryContext, jitContext->m_usage); + jitContext->m_subQueryContext = nullptr; } } -static void CleanupJitContextInner(JitContext* jitContext) +static void CleanupJitSubQueryContext( + JitSubQueryContext* subQueryContext, JitSubQueryExecState* subQueryExecState, bool isDropCachedPlan /* = false */) { - if (jitContext->m_innerTable != nullptr) { - if (jitContext->m_innerIndex != nullptr) { - if (jitContext->m_innerSearchKey != nullptr) { - jitContext->m_innerIndex->DestroyKey(jitContext->m_innerSearchKey); - jitContext->m_innerSearchKey = nullptr; - } - - if (jitContext->m_innerEndIteratorKey != nullptr) { - jitContext->m_innerIndex->DestroyKey(jitContext->m_innerEndIteratorKey); - jitContext->m_innerEndIteratorKey = nullptr; - } - - if (jitContext->m_innerBeginIterator != nullptr) { - destroyIterator(jitContext->m_innerBeginIterator); - jitContext->m_innerBeginIterator = nullptr; - } - - if (jitContext->m_innerEndIterator != nullptr) { - destroyIterator(jitContext->m_innerEndIterator); - jitContext->m_innerEndIterator = nullptr; - } - - jitContext->m_innerIndex = nullptr; - } - } -} - -static void CleanupJitContextSubQueryDataArray(JitContext* jitContext) -{ - if (jitContext->m_subQueryData != nullptr) { - MOT_LOG_TRACE("Cleaning up sub-query data array in JIT context %p", jitContext); - for (uint32_t i = 0; i < jitContext->m_subQueryCount; ++i) { - JitContext::SubQueryData* subQueryData = &jitContext->m_subQueryData[i]; - MOT_LOG_TRACE("Cleaning up sub-query %u data in JIT context %p", i, jitContext); - CleanupJitContextSubQueryData(subQueryData); - } - MOT::MemGlobalFree(jitContext->m_subQueryData); - jitContext->m_subQueryData = nullptr; - } -} - -static void CleanupJitContextSubQueryData(JitContext::SubQueryData* subQueryData) -{ - MemoryContext oldCtx = CurrentMemoryContext; - CurrentMemoryContext = SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR); - if (subQueryData->m_slot != nullptr) { - ExecDropSingleTupleTableSlot(subQueryData->m_slot); - subQueryData->m_slot = nullptr; - } - if (subQueryData->m_tupleDesc != nullptr) { - FreeTupleDesc(subQueryData->m_tupleDesc); - subQueryData->m_tupleDesc = nullptr; - } - if (subQueryData->m_index != nullptr) { - if (subQueryData->m_searchKey != nullptr) { - subQueryData->m_index->DestroyKey(subQueryData->m_searchKey); - subQueryData->m_searchKey = nullptr; - } - if (subQueryData->m_endIteratorKey != nullptr) { - subQueryData->m_index->DestroyKey(subQueryData->m_endIteratorKey); - subQueryData->m_endIteratorKey = nullptr; - } - subQueryData->m_index = nullptr; - } - CurrentMemoryContext = oldCtx; + CleanupSubQueryExecState(subQueryContext, subQueryExecState, isDropCachedPlan); + subQueryContext->m_index = nullptr; + subQueryContext->m_table = nullptr; } static JitContextPool* AllocSessionJitContextPool() @@ -814,11 +3766,13 @@ static JitContextPool* AllocSessionJitContextPool() extern void FreeSessionJitContextPool(JitContextPool* jitContextPool) { + MOT_LOG_TRACE( + "Destroy session-local JIT context pool, session context count: %u", u_sess->mot_cxt.jit_context_count); DestroyJitContextPool(jitContextPool); MOT::MemGlobalFree(jitContextPool); } -static MOT::Key* PrepareJitSearchKey(JitContext* jitContext, MOT::Index* index) +static MOT::Key* PrepareJitSearchKey(MotJitContext* jitContext, MOT::Index* index) { MOT::Key* key = index->CreateNewKey(); if (key == nullptr) { diff --git a/src/gausskernel/storage/mot/jit_exec/jit_exec.cpp b/src/gausskernel/storage/mot/jit_exec/jit_exec.cpp index bf6ee4558..c93e39df8 100644 --- a/src/gausskernel/storage/mot/jit_exec/jit_exec.cpp +++ b/src/gausskernel/storage/mot/jit_exec/jit_exec.cpp @@ -41,16 +41,19 @@ #include "utils/numeric.h" #include "utils/numeric_gs.h" #include "catalog/pg_aggregate.h" +#include "executor/spi.h" +#include "executor/executor.h" #include "mot_internal.h" #include "storage/mot/jit_exec.h" #include "jit_common.h" #include "jit_llvm_query_codegen.h" -#include "jit_tvm_query_codegen.h" +#include "jit_llvm_sp_codegen.h" #include "jit_source_pool.h" #include "jit_source_map.h" #include "jit_context_pool.h" #include "jit_plan.h" +#include "jit_plan_sp.h" #include "jit_statistics.h" #include "mot_engine.h" @@ -59,88 +62,429 @@ #include "mot_error.h" #include "utilities.h" #include "cycles.h" +#include "mot_atomic_ops.h" +#include "jit_profiler.h" -#include +#include namespace JitExec { DECLARE_LOGGER(LiteExecutor, JitExec); #ifdef MOT_JIT_TEST -static uint64_t totalExecCount = 0; +uint64_t totalQueryExecCount = 0; +uint64_t totalFunctionExecCount = 0; +static void UpdateTestStats(MotJitContext* jitContext, int newScan); #endif -extern JitPlan* IsJittable(Query* query, const char* queryString) -{ - JitPlan* jitPlan = NULL; - bool limitBreached = false; +static void JITXactCallback(XactEvent event, void* arg); +static MotJitContext* MakeDummyContext(JitSource* jitSource, JitContextUsage usage); +inline void PrepareSessionAccess() +{ // when running under thread-pool, it is possible to be executed from a thread that hasn't yet executed any MOT code // and thus lacking a thread id and NUMA node id. Nevertheless, we can be sure that a proper session context is set // up. - EnsureSafeThreadAccess(); + (void)EnsureSafeThreadAccess(); // since we use session local allocations and a session context might have not been created yet, we make sure such // one exists now - GetSafeTxn(__FUNCTION__); + (void)GetSafeTxn(__FUNCTION__); + + // make sure we get commit/rollback notifications in this session + if (!u_sess->mot_cxt.jit_xact_callback_registered) { + MOT_LOG_TRACE("Registering transaction call back for current session"); + RegisterXactCallback(JITXactCallback, nullptr); + u_sess->mot_cxt.jit_xact_callback_registered = true; + } +} + +static bool PrepareIsJittable(const char* queryName, bool isFunction) +{ + // since this might be called through initdb, we make a safety check here + if (MOT::MOTEngine::GetInstance() == nullptr) { + return false; + } + + // ensure all MOT thread/session-local identifiers are in place + PrepareSessionAccess(); // check limit not breached if (u_sess->mot_cxt.jit_context_count >= GetMotCodegenLimit()) { - MOT_LOG_TRACE("Query is not jittable: Reached the maximum of %d JIT contexts per session", + MOT_LOG_TRACE("%s %s is not jittable: Reached the maximum of %u JIT contexts per session", + isFunction ? "Stored procedure" : "query", + queryName, u_sess->mot_cxt.jit_context_count); - limitBreached = true; - } else if ((query->commandType == CMD_UPDATE) || (query->commandType == CMD_INSERT) || - (query->commandType == CMD_SELECT) || - (query->commandType == CMD_DELETE)) { // silently ignore other commands - // first check if already exists in global source - if (ContainsReadyCachedJitSource(queryString)) { - MOT_LOG_TRACE("Query JIT source already exists, reporting query is jittable"); - jitPlan = MOT_READY_JIT_PLAN; + JitStatisticsProvider::GetInstance().AddUnjittableLimitQuery(); + return false; + } + return true; +} +/* function name:IsJittableQuery + function purpose:Checks if the provided query is suitable for JIT processing, and prepares a JIT plan for it. + input:queryString The original string representation of the query.forcePlan If set to true, forces the creation of a new JIT plan even if one already exists. + output:Returns a JIT plan if the query is jittable, otherwise returns nullptr. + note:none + annotator:liushifa + annotate time:2023/10/05 22:22:47 + contact:3325287047@qq.com +*/ +extern JitPlan* IsJittableQuery(Query* query, const char* queryString, bool forcePlan /* = false */) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitPlan* jitPlan = nullptr; + volatile MemoryContext origCxt = CurrentMemoryContext; + + // push currently parsed query + void* prevQuery = u_sess->mot_cxt.jit_pg_query; + u_sess->mot_cxt.jit_pg_query = query; + + PG_TRY(); + { + if (!PrepareIsJittable(queryString, false)) { + MOT_LOG_TRACE("IsJittableQuery(): Failed to prepare for query parsing"); } else { - // either query never parsed or is expired, so we generate a plan - // print query - if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { - char* parsedQuery = nodeToString(query); - MOT_LOG_TRACE("Checking if query string is jittable: %s\nParsed query:\n%s", queryString, parsedQuery); - pfree(parsedQuery); + // silently ignore other commands + bool isSupported = ((query->commandType == CMD_UPDATE) || (query->commandType == CMD_INSERT) || + (query->commandType == CMD_SELECT) || (query->commandType == CMD_DELETE)); + if (isSupported) { + // first check if already exists in global source + if (!forcePlan && ContainsReadyCachedJitSource(queryString, false, true)) { + MOT_LOG_TRACE("Query JIT source already exists, reporting query is jittable"); + jitPlan = MOT_READY_JIT_PLAN; + } else { + // either query never parsed or is expired, so we generate a plan + // print query + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { + char* parsedQuery = nodeToString(query); + MOT_LOG_TRACE( + "Checking if query string is jittable: %s\nParsed query:\n%s\n", queryString, parsedQuery); + pfree(parsedQuery); + } + + // prepare a new plan + jitPlan = JitPreparePlan(query, queryString); + if (jitPlan != nullptr && jitPlan != MOT_READY_JIT_PLAN) { + if (JitPlanHasDistinct((JitPlan*)jitPlan)) { + MOT_LOG_TRACE("Enabling plan with DISTINCT aggregate"); + // In future, if we decide to disallow distinct operator, we just have to call + // JitDestroyPlan here. + } + + // plan generated so query is jittable + MOT_LOG_TRACE("Query is jittable by plan"); + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { + JitExplainPlan(query, (JitPlan*)jitPlan); + } + } + } } - // prepare a new plan - jitPlan = JitPreparePlan(query, queryString); - if (jitPlan != nullptr) { - // plan generated so query is jittable, but... - // we disqualify plan if we see distinct operator, until integrated with PG - if (JitPlanHasDistinct(jitPlan)) { - MOT_LOG_TRACE("Disqualifying plan with DISTINCT aggregate"); - JitDestroyPlan(jitPlan); - jitPlan = nullptr; + // update statistics + if (jitPlan == nullptr) { + MOT_LOG_TRACE("Query is not jittable: %s", queryString); + JitStatisticsProvider::GetInstance().AddUnjittableDisqualifiedQuery(); + } else { + // whether generating or cloning code, this is a jittable query + MOT_LOG_TRACE("Query is jittable: %s", queryString); + JitStatisticsProvider::GetInstance().AddJittableQuery(); + } + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while preparing JIT plan for query '%s': %s", queryString, edata->message); + ereport(WARNING, + (errmsg("Failed to parse query '%s' for MOT jitted execution.", queryString), + errdetail("%s", edata->detail))); + FlushErrorState(); + FreeErrorData(edata); + if (jitPlan != nullptr) { + JitDestroyPlan((JitPlan*)jitPlan); + jitPlan = nullptr; + } + } + PG_END_TRY(); + + // pop currently parsed query + u_sess->mot_cxt.jit_pg_query = prevQuery; + + if (jitPlan == nullptr) { + MOT_LOG_TRACE("Query is not jittable: %s", queryString); + } + return (JitPlan*)jitPlan; +} +/* function name:IsJittableFunction + function purpose:Checks if the given stored procedure (or function) is suitable for JIT processing + and prepares a JIT plan for it. + input:function The function (or stored procedure) to be checked.procTuple A tuple from the system catalog describing the function. functionOid The object identifier (OID) of the function. + forcePlan If set to true, forces the creation of a new JIT plan even if one already exists. + output:Returns a JIT plan if the function is jittable, otherwise returns nullptr. + note:none + annotator:liushifa + annotate time:2023/10/05 22:25:17 + contact:3325287047@qq.com +*/ +extern JitPlan* IsJittableFunction( + PLpgSQL_function* function, HeapTuple procTuple, Oid functionOid, bool forcePlan /* = false */) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile JitPlan* jitPlan = nullptr; + volatile bool nsPushed = false; + MOT::mot_string qualifiedFunctionName; + volatile const char* qualifiedFunctionNameStr = ""; + char* functionName = ""; + + ++u_sess->mot_cxt.jit_compile_depth; // raise flag to guard SPI calls + u_sess->mot_cxt.jit_parse_error = 0; + PG_TRY(); + { + bool procNameIsNull = false; + bool procSrcIsNull = false; + bool procIsStrictIsNull = false; + Datum procNameDatum = SysCacheGetAttr(PROCOID, procTuple, Anum_pg_proc_proname, &procNameIsNull); + Datum procSrcDatum = SysCacheGetAttr(PROCOID, procTuple, Anum_pg_proc_prosrc, &procSrcIsNull); + Datum procIsStrictDatum = SysCacheGetAttr(PROCOID, procTuple, Anum_pg_proc_proisstrict, &procIsStrictIsNull); + bool hasNullAttr = (procNameIsNull || procSrcIsNull || procIsStrictIsNull); + functionName = NameStr(*DatumGetName(procNameDatum)); + if (!PrepareIsJittable(functionName, true)) { + MOT_LOG_TRACE("IsJittableQuery(): Failed to prepare for query parsing"); + } else if (hasNullAttr) { + MOT_LOG_TRACE( + "Stored procedure is not jittable: catalog entry for stored procedure contains null attributes"); + } else { + // first check if already exists in global source (all functions defined in global name space) + if (!qualifiedFunctionName.format("%s.%u", functionName, (unsigned)functionOid)) { + MOT_LOG_TRACE("Failed to format qualified function name"); + } else { + qualifiedFunctionNameStr = qualifiedFunctionName.c_str(); + if (!forcePlan && ContainsReadyCachedJitSource((const char*)qualifiedFunctionNameStr, true, true)) { + MOT_LOG_TRACE("Stored procedure JIT source already exists, reporting stored procedure is jittable"); + jitPlan = MOT_READY_JIT_PLAN; } else { - MOT_LOG_TRACE("Query %s jittable by plan", jitPlan ? "is" : "is not"); - if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { - JitExplainPlan(query, jitPlan); + // either function never parsed or is expired, so we generate a plan + if (PushJitSourceNamespace(functionOid, (const char*)qualifiedFunctionNameStr)) { + nsPushed = true; + + // get required function attributes + char* functionSource = TextDatumGetCString(procSrcDatum); + bool isStrict = DatumGetBool(procIsStrictDatum); + + // print query + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { + MOT_LOG_TRACE("Checking if function '%s' is jittable:\n%s\n", + qualifiedFunctionNameStr, + functionSource); + plpgsql_dumptree(function); + } + + // prepare a new plan + jitPlan = JitPrepareFunctionPlan(function, functionSource, functionName, functionOid, isStrict); + if (jitPlan != nullptr) { + MOT_LOG_TRACE("Function %s is jittable by plan", qualifiedFunctionNameStr); + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { + JitExplainPlan(function, (JitPlan*)jitPlan); + } + } + PopJitSourceNamespace(); + nsPushed = false; } } + + // update statistics + if (jitPlan == nullptr) { + MOT_LOG_TRACE("Function is not jittable: %s", qualifiedFunctionNameStr); + JitStatisticsProvider::GetInstance().AddUnjittableDisqualifiedQuery(); + } else { + // whether generating or cloning code, this is a jittable query + MOT_LOG_TRACE("Function is jittable: %s", qualifiedFunctionNameStr); + JitStatisticsProvider::GetInstance().AddJittableQuery(); + } } } } - - MOT_LOG_TRACE("Query %s jittable: %s", (jitPlan != nullptr) ? "is" : "is not", queryString); - - // update statistics - if (limitBreached) { - MOT_ASSERT(jitPlan == nullptr); - JitStatisticsProvider::GetInstance().AddUnjittableLimitQuery(); - } else if (jitPlan == nullptr) { - JitStatisticsProvider::GetInstance().AddUnjittableDisqualifiedQuery(); - } else { - // whether generating or cloning code, this is a jittable query - JitStatisticsProvider::GetInstance().AddJittableQuery(); + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while preparing JIT plan for stored procedure '%s': %s", + qualifiedFunctionNameStr, + edata->message); + ereport(WARNING, + (errmsg("Failed to parse function '%s' for MOT jitted execution.", qualifiedFunctionNameStr), + errdetail("%s", edata->detail))); + FlushErrorState(); + FreeErrorData(edata); + if (nsPushed) { + PopJitSourceNamespace(); + } + if (jitPlan != nullptr) { + JitDestroyPlan((JitPlan*)jitPlan); + jitPlan = nullptr; + } } + PG_END_TRY(); - return jitPlan; + // decrement SPI guard counter + --u_sess->mot_cxt.jit_compile_depth; + + if (jitPlan == nullptr) { + if (u_sess->mot_cxt.jit_parse_error != 0) { + MOT_LOG_WARN("Stored procedure is not jittable: %s", qualifiedFunctionNameStr); + } else { + MOT_LOG_TRACE("Stored procedure is not jittable: %s", qualifiedFunctionNameStr); + } + } + return (JitPlan*)jitPlan; } -static void ProcessJitResult(MOT::RC result, JitContext* jitContext, int newScan) +extern bool IsInvokeReadyFunction(Query* query) { + // if function code generation is altogether disabled, then we silently fail + if (!IsMotSPCodegenEnabled()) { + return false; + } + if (query->commandType != CMD_SELECT) { + MOT_LOG_DEBUG("IsInvokeReadyFunction(): Disqualifying invoke query - not a SELECT command"); + return false; + } + if (!CheckQueryAttributes(query, false, false, false)) { + MOT_LOG_DEBUG("IsInvokeReadyFunction(): Disqualifying invoke query - Invalid query attributes"); + return false; + } + if ((query->jointree) && (query->jointree->fromlist || query->jointree->quals)) { + MOT_LOG_DEBUG("IsInvokeReadyFunction(): Disqualifying invoke query - FROM clause is not empty"); + return false; + } + if (list_length(query->targetList) != 1) { + MOT_LOG_DEBUG( + "IsInvokeReadyFunction(): Disqualifying invoke query - target list does not contain exactly one entry"); + return false; + } + + TargetEntry* targetEntry = (TargetEntry*)linitial(query->targetList); + if (targetEntry->expr->type != T_FuncExpr) { + MOT_LOG_DEBUG( + "IsInvokeReadyFunction(): Disqualifying invoke query - single target entry is not a function expression"); + return false; + } + + // get function name first + FuncExpr* funcExpr = (FuncExpr*)targetEntry->expr; + char* funcName = get_func_name(funcExpr->funcid); + if (funcName == nullptr) { + MOT_LOG_TRACE("IsInvokeReadyFunction(): Disqualifying invoke query - Failed to find function name for function " + "id %u", + (unsigned)funcExpr->funcid); + return false; + } + MOT::mot_string qualifiedFunctionName; + if (!qualifiedFunctionName.format("%s.%u", funcName, (unsigned)funcExpr->funcid)) { + MOT_LOG_TRACE("IsInvokeReadyFunction(): Disqualifying invoke query - Failed to format qualified function name " + "for function %s", + funcName); + pfree(funcName); + return false; + } + bool result = ContainsReadyCachedJitSource(qualifiedFunctionName.c_str(), true); + if (!result) { + MOT_LOG_TRACE("IsInvokeReadyFunction(): Disqualifying invoke query - could not find a ready context for " + "function %s", + funcName); + } + pfree(funcName); + return result; +} + +static void ProcessErrorResult(MOT::RC result, MotJitContext* jitContext, int newScan) +{ + if (result == MOT::RC_JIT_SP_EXCEPTION) { + if (IsJitSubContextInline(jitContext)) { +#ifdef MOT_JIT_TEST + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY && newScan) { + UpdateTestStats(jitContext, newScan); + } +#endif + return; + } + + const char* errorDetail = nullptr; + const char* errorHint = nullptr; + bytea* txt = DatumGetByteaP(jitContext->m_execState->m_errorMessage); + const char* errorMessage = (const char*)VARDATA(txt); + if (jitContext->m_execState->m_errorDetail != 0) { + bytea* txtd = DatumGetByteaP(jitContext->m_execState->m_errorDetail); + errorDetail = (const char*)VARDATA(txtd); + } + if (jitContext->m_execState->m_errorHint != 0) { + bytea* txth = DatumGetByteaP(jitContext->m_execState->m_errorHint); + errorHint = (const char*)VARDATA(txth); + } + + RaiseEreport(jitContext->m_execState->m_sqlState, errorMessage, errorDetail, errorHint); + } + + if (result != MOT::RC_OK) { + void* arg1 = nullptr; + void* arg2 = nullptr; + + MOT::TxnManager* currTxn = u_sess->mot_cxt.jit_txn; + + // prepare message argument for error report + ColumnDef stub; + if (result == MOT::RC_UNIQUE_VIOLATION) { + arg1 = (void*)(currTxn->m_errIx ? currTxn->m_errIx->GetName().c_str() : ""); + arg2 = (void*)(currTxn->m_errMsgBuf); + } else if (result == MOT::RC_NULL_VIOLATION) { + MOT::Table* table = jitContext->m_execState->m_nullViolationTable; + stub.colname = table->GetField((uint64_t)jitContext->m_execState->m_nullColumnId)->m_name; + arg1 = (void*)&stub; + arg2 = (void*)table; + } + + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { + MOT_LOG_ERROR_STACK("Failed to execute jitted function with error code: %d (%s), query is: %s", + (int)result, + MOT::RcToString(result), + jitContext->m_queryString); + } + if (result == MOT::RC_ABORT) { + JitStatisticsProvider::GetInstance().AddAbortExecQuery(); + } else { + JitStatisticsProvider::GetInstance().AddFailExecQuery(); + } + report_pg_error(result, (void*)arg1, (void*)arg2); + } +} + +static void ProcessJitResult(MOT::RC result, MotJitContext* jitContext, int newScan) +{ + // the result is being returned from jitted query + if (result == MOT::RC_LOCAL_ROW_NOT_FOUND && jitContext->m_rc != MOT::RC_OK) { + switch (jitContext->m_rc) { + case MOT::RC_SERIALIZATION_FAILURE: + case MOT::RC_MEMORY_ALLOCATION_ERROR: + case MOT::RC_ABORT: + result = jitContext->m_rc; + jitContext->m_rc = MOT::RC_OK; + break; + + default: + MOT_LOG_ERROR("ProcessJitResult(): Unsupported error (%s), changing result to ABORT", + RcToString(jitContext->m_rc)); + MOT_ASSERT(false); + result = MOT::RC_ABORT; + jitContext->m_rc = MOT::RC_OK; + } + } + + // we want to make sure that jitContext->m_rc was already handled. jitContext->m_rc != RC_OK is an unexpected + // behavior. + MOT_ASSERT(jitContext->m_rc == MOT::RC_OK); // NOTE: errors might be reported in a better way, so this part can be reviewed sometime // we ignore "local row not found" in SELECT and DELETE scenarios if (result == MOT::RC_LOCAL_ROW_NOT_FOUND) { @@ -154,12 +498,12 @@ static void ProcessJitResult(MOT::RC result, JitContext* jitContext, int newScan case JIT_COMMAND_COMPOUND_SELECT: case JIT_COMMAND_UPDATE: case JIT_COMMAND_RANGE_UPDATE: + case JIT_COMMAND_RANGE_DELETE: // this is considered as successful execution JitStatisticsProvider::GetInstance().AddInvokeQuery(); if (newScan) { #ifdef MOT_JIT_TEST - MOT_ATOMIC_INC(totalExecCount); - MOT_LOG_INFO("JIT total queries executed: %" PRIu64, MOT_ATOMIC_LOAD(totalExecCount)); + UpdateTestStats(jitContext, newScan); #endif JitStatisticsProvider::GetInstance().AddExecQuery(); } @@ -169,118 +513,216 @@ static void ProcessJitResult(MOT::RC result, JitContext* jitContext, int newScan } } - const char* arg1 = ""; - const char* arg2 = ""; - - MOT::TxnManager* currTxn = u_sess->mot_cxt.jit_txn; - - // prepare message argument for error report - if (result == MOT::RC_UNIQUE_VIOLATION) { - arg1 = currTxn->m_errIx ? currTxn->m_errIx->GetName().c_str() : ""; - arg2 = currTxn->m_errMsgBuf; - } else if (result == MOT::RC_NULL_VIOLATION) { - arg1 = jitContext->m_table->GetField((uint64_t)jitContext->m_nullColumnId)->m_name; - arg2 = jitContext->m_table->GetLongTableName().c_str(); - } - - if (result != MOT::RC_OK) { - if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { - MOT_LOG_ERROR_STACK( - "Failed to execute jitted function with error code: %d (%s)", (int)result, MOT::RcToString(result)); - } - if (result == MOT::RC_ABORT) { - JitStatisticsProvider::GetInstance().AddAbortExecQuery(); - } else { - JitStatisticsProvider::GetInstance().AddFailExecQuery(); - } - report_pg_error(result, (void*)arg1, (void*)arg2); - } + ProcessErrorResult(result, jitContext, newScan); } -static JitContext* GenerateJitContext(Query* query, const char* queryString, JitPlan* jitPlan, JitSource* jitSource) +static MotJitContext* ProcessGeneratedJitContext(JitSource* jitSource, MotJitContext* sourceJitContext, + const char* queryString, JitCodegenStats* codegenStats, JitContextUsage usage) { - JitContext* jitContext = nullptr; - JitContext* sourceJitContext = nullptr; - - uint64_t startTime = GetSysClock(); - if (g_instance.mot_cxt.jitExecMode == JIT_EXEC_MODE_LLVM) { - MOT_LOG_TRACE("Generating LLVM JIT context for query: %s", queryString); - sourceJitContext = JitCodegenLlvmQuery(query, queryString, jitPlan); - } else { - MOT_ASSERT(g_instance.mot_cxt.jitExecMode == JIT_EXEC_MODE_TVM); - MOT_LOG_TRACE("Generating TVM JIT context for query: %s", queryString); - sourceJitContext = JitCodegenTvmQuery(query, queryString, jitPlan); - } - uint64_t endTime = GetSysClock(); - + JitCodegenState newState = JitCodegenState::JIT_CODEGEN_NONE; if (sourceJitContext == nullptr) { // notify error for all waiters - this query will never again be JITTed - cleanup only during database shutdown - MOT_LOG_TRACE("Failed to generate code for query, signaling error context for query: %s", queryString); - SetJitSourceError(jitSource, MOT::GetRootError()); - JitStatisticsProvider::GetInstance().AddCodeGenErrorQuery(); - } else { - MOT_LOG_TRACE("Generated JIT context %p for query: %s", sourceJitContext, queryString); - MOT_LOG_TRACE("Cloning ready source context %p", sourceJitContext); - jitContext = CloneJitContext(sourceJitContext); - if (jitContext != nullptr) { - if (SetJitSourceReady(jitSource, sourceJitContext)) { - MOT_LOG_TRACE("Installed ready JIT context %p for query: %s", sourceJitContext, queryString); - ++u_sess->mot_cxt.jit_context_count; - AddJitSourceContext(jitSource, jitContext); // register for cleanup due to DDL - jitContext->m_jitSource = jitSource; - jitContext->m_queryString = jitSource->_query_string; - // update statistics - MOT_LOG_TRACE("Registered JIT context %p in JIT source %p for cleanup", jitContext, jitSource); - JitStatisticsProvider& instance = JitStatisticsProvider::GetInstance(); - instance.AddCodeGenTime(MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime)); - instance.AddCodeGenQuery(); - instance.AddCodeCloneQuery(); - } else { - // this is illegal state transition error in JIT source (internal bug) - // there is already a JIT context present in the JIT source (maybe generated by another session, - // although impossible), so we just fail JIT for this session (other sessions may still benefit from - // existing JIT context). Pay attention that we cannot replace the JIT context in the JIT source, since - // the JIT function in it is probably still being used by other sessions - MOT_REPORT_ERROR(MOT_ERROR_INVALID_STATE, - "JIT Compile", - "Failed to set ready source context %p, disqualifying query: %s", - sourceJitContext, - queryString); - DestroyJitContext(sourceJitContext); - DestroyJitContext(jitContext); - jitContext = nullptr; - JitStatisticsProvider::GetInstance().AddCodeGenErrorQuery(); - } - } else { - // this is not expected to happen (because the number of context objects per session equals number of JIT - // source objects), but we still must set JIT source state to error, otherwise all waiting sessions will - // never wakeup - SetJitSourceError(jitSource, MOT::GetRootError()); - MOT_REPORT_ERROR(MOT_ERROR_RESOURCE_UNAVAILABLE, - "JIT Compile", - "Failed to clone ready source context %p, disqualifying query: %s", - sourceJitContext, - queryString); - DestroyJitContext(sourceJitContext); - JitStatisticsProvider::GetInstance().AddCodeCloneErrorQuery(); + MOT_LOG_TRACE("Failed to generate code for query with JIT source %p, signaling error context for query: %s", + jitSource, + queryString); + SetJitSourceError(jitSource, MOT::GetRootError(), &newState); + if (newState == JitCodegenState::JIT_CODEGEN_DEPRECATE) { + // ATTENTION: Do not use the jitSource after this point, as it might be removed from the deprecated list + // and freed by other sessions. + CleanupConcurrentlyDroppedSPSource(queryString); } + JitStatisticsProvider::GetInstance().AddCodeGenErrorQuery(); + return nullptr; } + MOT_LOG_TRACE("Generated JIT context %p for query: '%s', codegenTime: %" PRIu64 " micros", + sourceJitContext, + queryString, + codegenStats->m_codegenTime); + + bool isSPGlobalSource = false; + if ((sourceJitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) && + (jitSource->m_usage == JIT_CONTEXT_GLOBAL)) { + isSPGlobalSource = true; + } + + if (!SetJitSourceReady(jitSource, sourceJitContext, codegenStats, &newState)) { + // Two possible causes: + // 1. Stored procedure was dropped and deprecated when we were compiling. In this case, we just fail + // JIT code generation to trigger re-validation. + // 2. Illegal state transition error in JIT source (internal bug), there is already a JIT context present + // in the JIT source (maybe generated by another session, although impossible). + // So we just fail JIT for this session (other sessions may still benefit from existing JIT context). + // Pay attention that we cannot replace the JIT context in the JIT source, since + // the JIT function in it is probably still being used by other sessions + MOT_REPORT_ERROR(MOT_ERROR_INVALID_STATE, + "JIT Compile", + "Failed to set ready source context %p to JIT source %p (newState %s), disqualifying query: %s", + sourceJitContext, + jitSource, + JitCodegenStateToString(newState), + queryString); + if (newState == JitCodegenState::JIT_CODEGEN_DEPRECATE) { + // ATTENTION: Do not use the jitSource after this point, as it might be removed from the deprecated list + // and freed by other sessions. + CleanupConcurrentlyDroppedSPSource(queryString); + } + DestroyJitContext(sourceJitContext); + JitStatisticsProvider::GetInstance().AddCodeGenErrorQuery(); + return nullptr; + } + + MOT_LOG_TRACE("Installed ready JIT source context %p to JIT source %p (newState %s) for query: %s", + sourceJitContext, + jitSource, + JitCodegenStateToString(newState), + queryString); + + bool needPruning = false; + MotJitContext* jitContext = nullptr; + if (isSPGlobalSource) { + // If this is a global SP source, lock the source map for pruning. + LockJitSourceMap(); + } + + LockJitSource(jitSource); + // attention: the source context has its JIT source set up properly, so clone can use it + JitCodegenState codegenState = jitSource->m_codegenState; + if (jitSource->m_sourceJitContext == sourceJitContext) { + if (codegenState == JitCodegenState::JIT_CODEGEN_READY) { + if ((jitSource->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) && + (jitSource->m_usage == JIT_CONTEXT_GLOBAL)) { + needPruning = true; + MOT_ASSERT(isSPGlobalSource); + } + jitContext = CloneJitContext(sourceJitContext, usage); + } else if (codegenState == JitCodegenState::JIT_CODEGEN_UNAVAILABLE || + codegenState == JitCodegenState::JIT_CODEGEN_PENDING) { + jitContext = MakeDummyContext(jitSource, usage); + } else { + MOT_LOG_TRACE("Cannot clone context from JIT source %p after code-generation, unexpected state %s: %s", + jitSource, + JitCodegenStateToString(codegenState), + jitSource->m_queryString); + } + } else { + MOT_LOG_TRACE("JIT source %p (codegenState %s) context %p changed concurrently to %p after code-generation", + jitSource, + JitCodegenStateToString(codegenState), + sourceJitContext, + jitSource->m_sourceJitContext); + } + UnlockJitSource(jitSource); + + if (isSPGlobalSource) { + if (needPruning) { + MOT_LOG_TRACE("Pruning global ready function source %p after code-generation: %s", + jitSource, + jitSource->m_queryString); + PruneNamespace(jitSource); + } + UnlockJitSourceMap(); + } + + if (jitContext == nullptr) { + // we keep the source JIT context intact in the JIT source, maybe some time later we will have resources for + // cloning. + MOT_REPORT_ERROR(MOT_ERROR_RESOURCE_UNAVAILABLE, + "JIT Compile", + "Failed to clone ready source context %p from JIT source %p (codegenState %s), disqualifying query: %s", + sourceJitContext, + jitSource, + JitCodegenStateToString(codegenState), + queryString); + JitStatisticsProvider::GetInstance().AddCodeCloneErrorQuery(); + return nullptr; + } + + MOT_LOG_TRACE("Registered JIT context %p in JIT source %p for cleanup", jitContext, jitSource); + + // update statistics + JitStatisticsProvider& instance = JitStatisticsProvider::GetInstance(); + instance.AddCodeGenTime(codegenStats->m_codegenTime); + instance.AddCodeGenQuery(); + instance.AddCodeCloneQuery(); + return jitContext; } -extern JitContext* JitCodegenQuery(Query* query, const char* queryString, JitPlan* jitPlan) +static MotJitContext* GenerateQueryJitContext( + Query* query, const char* queryString, JitPlan* jitPlan, JitSource* jitSource, JitContextUsage usage) { - JitContext* jitContext = nullptr; + MotJitContext* sourceJitContext = nullptr; + JitCodegenStats codegenStats = {}; - MOT_LOG_TRACE("*** Generating cached code for query: %s", queryString); + uint64_t startTime = GetSysClock(); + MOT_LOG_TRACE("Generating LLVM JIT context for query: %s", queryString); + sourceJitContext = JitCodegenLlvmQuery(query, queryString, jitPlan, codegenStats); + uint64_t endTime = GetSysClock(); + uint64_t timeMicros = MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime); + codegenStats.m_codegenTime = timeMicros; + + return ProcessGeneratedJitContext(jitSource, sourceJitContext, queryString, &codegenStats, usage); +} + +static MotJitContext* GenerateFunctionJitContext(PLpgSQL_function* function, HeapTuple procTuple, Oid functionOid, + ReturnSetInfo* returnSetInfo, const char* queryString, JitPlan* jitPlan, JitSource* jitSource, + JitContextUsage usage) +{ + MotJitContext* sourceJitContext = nullptr; + JitCodegenStats codegenStats = {}; + uint64_t startTime = GetSysClock(); + MOT_LOG_TRACE("Generating LLVM JIT context for function: %s", queryString); + sourceJitContext = JitCodegenLlvmFunction(function, procTuple, functionOid, returnSetInfo, jitPlan, codegenStats); + uint64_t endTime = GetSysClock(); + uint64_t timeMicros = MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime); + codegenStats.m_codegenTime = timeMicros; + + return ProcessGeneratedJitContext(jitSource, sourceJitContext, queryString, &codegenStats, usage); +} + +static MotJitContext* MakeDummyContext(JitSource* jitSource, JitContextUsage usage) +{ + JitQueryContext* jitContext = (JitQueryContext*)AllocJitContext(usage, jitSource->m_contextType); + if (jitContext != nullptr) { + jitContext->m_contextType = jitSource->m_contextType; + jitContext->m_commandType = jitSource->m_commandType; + jitContext->m_validState = JIT_CONTEXT_PENDING_COMPILE; + AddJitSourceContext(jitSource, jitContext); + } + return jitContext; +} + +static void SetupJitSourceForCodegen(JitSource* jitSource, JitContextType contextType, JitPlan* jitPlan) +{ + LockJitSource(jitSource); + MOT_ASSERT(jitSource->m_codegenState == JitCodegenState::JIT_CODEGEN_UNAVAILABLE); + MOT_ASSERT(jitPlan != MOT_READY_JIT_PLAN); // we must have a ready plan + jitSource->m_contextType = contextType; + jitSource->m_commandType = jitPlan->_command_type; + if ((jitSource->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) && + (jitSource->m_commandType == JIT_COMMAND_INVOKE)) { + jitSource->m_functionOid = ((JitInvokePlan*)jitPlan)->_function_id; + } else if (jitSource->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + jitSource->m_functionOid = ((JitFunctionPlan*)jitPlan)->_function_id; + } + UnlockJitSource(jitSource); +} + +template +static MotJitContext* JitCodegen( + const char* queryName, JitPlan* jitPlan, Generator& generator, bool globalNamespace, JitContextUsage usage) +{ + MotJitContext* jitContext = nullptr; + + MOT_LOG_TRACE("*** Generating cached code for query: %s", queryName); MOT_ASSERT(jitPlan); // search for a cached JIT source - either use an existing one or generate a new one JitSource* jitSource = nullptr; do { // instead of goto LockJitSourceMap(); // jit-source already exists, so wait for it to become ready - jitSource = GetCachedJitSource(queryString); + jitSource = GetCachedJitSource(queryName, globalNamespace); if (jitSource != nullptr) { MOT_LOG_TRACE("Found a jit-source %p", jitSource); UnlockJitSourceMap(); @@ -288,204 +730,1401 @@ extern JitContext* JitCodegenQuery(Query* query, const char* queryString, JitPla // ATTENTION: JIT source cannot get expired at this phase, since DDL statements (that might cause the // JIT Source to expire) cannot run in parallel with other statements // Note: cached context was found, but maybe it is still being compiled, so we wait for it to be ready - MOT_LOG_TRACE("Waiting for context to be ready: %s", queryString); - JitContextStatus ctxStatus = WaitJitContextReady(jitSource, &jitContext); - if (ctxStatus == JIT_CONTEXT_READY) { - MOT_LOG_TRACE("Context is ready: %s", queryString); - } else if (ctxStatus == JIT_CONTEXT_EXPIRED) { // need to regenerate context - // generate JIT context, install it and notify - MOT_LOG_TRACE("Regenerating real JIT code for expired context: %s", queryString); + MOT_LOG_TRACE("Waiting for context to be ready: %s", queryName); + JitCodegenState codegenState = GetReadyJitContext(jitSource, &jitContext, usage, jitPlan); + if (codegenState == JitCodegenState::JIT_CODEGEN_READY) { + MOT_LOG_TRACE("Collected ready context: %s", queryName); + } else if (codegenState == JitCodegenState::JIT_CODEGEN_EXPIRED) { + // regenerate JIT context, install it and notify + MOT_LOG_TRACE("Regenerating real JIT code for expired context: %s", queryName); // we must prepare the analysis variables again (because we did not call IsJittable()) // in addition, table/index definition might have change so we must re-analyze - if (jitPlan != MOT_READY_JIT_PLAN) { - JitDestroyPlan(jitPlan); + if (jitPlan == MOT_READY_JIT_PLAN) { + jitPlan = generator.IsJittable(); } - jitPlan = IsJittable(query, queryString); if (jitPlan == nullptr) { - MOT_LOG_TRACE("Failed to re-analyze expired JIT source, notifying error status: %s", queryString); + MOT_LOG_TRACE("Failed to re-analyze expired JIT source, notifying error status: %s", queryName); SetJitSourceError(jitSource, MOT_ERROR_INTERNAL); } else { - jitContext = GenerateJitContext(query, queryString, jitPlan, jitSource); + JitFunctionPlan* functionPlan = nullptr; + if (JitPlanHasFunctionPlan(jitPlan, &functionPlan)) { + // fn_xmin equal to m_functionTxnId is possible, if the JIT source moved from ERROR to + // UNAVAILABLE/EXPIRED in GetReadyJitContext. + MOT_ASSERT((functionPlan->m_function->fn_xmin >= jitSource->m_functionTxnId) && + (functionPlan->m_function->fn_xmin >= jitSource->m_expireTxnId)); + } + SetupJitSourceForCodegen(jitSource, generator.GetContextType(), jitPlan); + jitContext = generator.GenerateJitContext(jitSource, jitPlan, usage); } - } else { // code generation (by another session) failed - MOT_LOG_TRACE("Cached context status is not ready: %s", queryString); + } else if ((codegenState == JitCodegenState::JIT_CODEGEN_UNAVAILABLE) || + (codegenState == JitCodegenState::JIT_CODEGEN_PENDING)) { + // prepare a dummy context + jitContext = MakeDummyContext(jitSource, usage); + MOT_LOG_TRACE("Generated dummy context %p for query: %s", jitContext, queryName); + } else { // code generation (by another session) failed, or source was deprecated + MOT_LOG_TRACE("Cached context status is not ready: %s", queryName); break; // goto cleanup } - } else { // jit-source is not ready, so we need to generate code - MOT_ASSERT(jitPlan != MOT_READY_JIT_PLAN); // we must have a real plan, right? - MOT_LOG_TRACE("JIT-source not found, generating code for query: %s", queryString); - if (GetJitSourceMapSize() == GetMotCodegenLimit()) { - MOT_LOG_DEBUG("Skipping query code generation: Reached total maximum of JIT source objects %d", + } else { + // JIT source is not ready, so we need to generate code. + MOT_LOG_TRACE("JIT-source not found, generating code for query: %s", queryName); + + if (generator.GetContextType() == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + JitFunctionPlan* functionPlan = (JitFunctionPlan*)jitPlan; + Oid functionOid = functionPlan->_function_id; + if (IsDroppedSP(functionOid)) { + MOT_LOG_ERROR("Function is already deprecated, skipping premature re-validation: %s", queryName); + UnlockJitSourceMap(); + break; // goto cleanup + } + } + + // we allocate an empty cached entry and install it in the global map, other threads can wait until it + // is ready (thus only 1 thread regenerates code) + JitContextUsage sourceUsage = + (u_sess->mot_cxt.jit_session_source_map != nullptr) ? JIT_CONTEXT_LOCAL : JIT_CONTEXT_GLOBAL; + jitSource = AllocJitSource(queryName, sourceUsage); + if (jitSource == nullptr) { + MOT_LOG_TRACE("Skipping query code generation: Reached total maximum of JIT source objects %d", GetMotCodegenLimit()); UnlockJitSourceMap(); break; // goto cleanup } - // we allocate an empty cached entry and install it in the global map, other threads can wait until it - // is ready (thus only 1 thread regenerates code) - jitSource = AllocPooledJitSource(queryString); - if (jitSource != nullptr) { - MOT_LOG_TRACE("Created jit-source object %p", jitSource); - if (!AddCachedJitSource(jitSource)) { // unexpected: entry already exists (this is a bug) - MOT_LOG_TRACE("Failed to add jit-source object to map"); - UnlockJitSourceMap(); - FreePooledJitSource(jitSource); - break; // goto cleanup - } - UnlockJitSourceMap(); // let other operations continue - // generate JIT context, install it and notify - MOT_LOG_TRACE("Generating JIT code"); - jitContext = GenerateJitContext(query, queryString, jitPlan, jitSource); + MOT_ASSERT(jitPlan != MOT_READY_JIT_PLAN); // we must have a ready plan + + SetupJitSourceForCodegen(jitSource, generator.GetContextType(), jitPlan); + + MOT_LOG_TRACE( + "Created jit-source object %p (contextType %s, commandType %s, functionOid %u) during code-gen: %s", + jitSource, + JitContextTypeToString(jitSource->m_contextType), + CommandToString(jitSource->m_commandType), + jitSource->m_functionOid, + queryName); + + if (!AddCachedJitSource(jitSource, globalNamespace)) { + // unexpected: entry already exists (this is a bug) + MOT_LOG_TRACE("Failed to add jit-source object to map"); + UnlockJitSourceMap(); + DestroyJitSource(jitSource); + jitSource = nullptr; + break; // goto cleanup } + + UnlockJitSourceMap(); // let other operations continue + + // generate JIT context, install it and notify + MOT_LOG_TRACE("Generating JIT code"); + jitContext = generator.GenerateJitContext(jitSource, jitPlan, usage); } } while (0); - // cleanup - if ((jitPlan != nullptr) && (jitPlan != MOT_READY_JIT_PLAN)) { - JitDestroyPlan(jitPlan); - } - return jitContext; } -extern void JitResetScan(JitContext* jitContext) +struct JitQueryGenerator { +public: + JitQueryGenerator(Query* query, const char* queryString) : m_query(query), m_queryString(queryString) + {} + + inline JitPlan* IsJittable() + { + return IsJittableQuery(m_query, m_queryString, true); + } + + inline MotJitContext* GenerateJitContext(JitSource* jitSource, JitPlan* jitPlan, JitContextUsage usage) + { + return GenerateQueryJitContext(m_query, m_queryString, jitPlan, jitSource, usage); + } + + inline JitContextType GetContextType() const + { + return JitContextType::JIT_CONTEXT_TYPE_QUERY; + } + +private: + Query* m_query; + const char* m_queryString; +}; + +extern MotJitContext* JitCodegenQuery(Query* query, const char* queryString, JitPlan* jitPlan, JitContextUsage usage) { - MOT_LOG_DEBUG("JitResetScan(): Resetting iteration count for context %p", jitContext); - jitContext->m_iterCount = 0; + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile MotJitContext* result = nullptr; + + // push currently parsed query + void* prevQuery = u_sess->mot_cxt.jit_pg_query; + u_sess->mot_cxt.jit_pg_query = query; + u_sess->mot_cxt.jit_codegen_error = 0; + + PG_TRY(); + { + MOT_LOG_TRACE("Generating code for query: %s", queryString); + uint64_t startTime = GetSysClock(); + JitQueryGenerator generator(query, queryString); + result = JitCodegen(queryString, jitPlan, generator, false, usage); + if (result != nullptr) { + uint64_t endTime = GetSysClock(); + uint64_t timeMicros = MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime); + MOT_LOG_TRACE("Code generation time %" PRIu64 " micros for query: %s", timeMicros, queryString); + } + } + PG_CATCH(); + { + // cleanup + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while generating code for query '%s': %s", queryString, edata->message); + ereport(WARNING, + (errmsg("Failed to generate code for query '%s' for MOT jitted execution.", queryString), + errdetail("%s", edata->detail))); + u_sess->mot_cxt.jit_codegen_error = edata->sqlerrcode; + FreeErrorData(edata); + FlushErrorState(); + } + PG_END_TRY(); + + // pop currently parsed query + u_sess->mot_cxt.jit_pg_query = prevQuery; + + if (result == nullptr) { + MOT_LOG_WARN("Failed to JIT-compile query: %s", queryString); + } + return (MotJitContext*)result; } -extern int JitExecQuery( - JitContext* jitContext, ParamListInfo params, TupleTableSlot* slot, uint64_t* tuplesProcessed, int* scanEnded) +extern MotJitContext* TryJitCodegenQuery(Query* query, const char* queryString) { - int result = 0; + MotJitContext* jitContext = nullptr; + // make sure SPI is set-up on top level call + SPIAutoConnect spiAutoConn; + if (!spiAutoConn.IsConnected()) { + int rc = spiAutoConn.GetErrorCode(); + MOT_LOG_TRACE("Failed to connect to SPI while generating code for query: %s (%u)", + queryString, + SPI_result_code_string(rc), + rc); + return nullptr; + } + + JitPlan* jitPlan = IsJittableQuery(query, queryString); + if (jitPlan != nullptr) { + jitContext = JitCodegenQuery(query, queryString, jitPlan, JIT_CONTEXT_LOCAL); + if (jitContext == nullptr) { + MOT_LOG_TRACE("Failed to generate jitted MOT code for query: %s", queryString); + } + JitDestroyPlan(jitPlan); + } + return jitContext; +} + +struct JitFunctionGenerator { +public: + JitFunctionGenerator(PLpgSQL_function* function, HeapTuple procTuple, Oid functionOid, ReturnSetInfo* returnSetInfo, + const char* queryString) + : m_function(function), + m_procTuple(procTuple), + m_functionOid(functionOid), + m_returnSetInfo(returnSetInfo), + m_queryString(queryString) + {} + + inline JitPlan* IsJittable() + { + return IsJittableFunction(m_function, m_procTuple, m_functionOid, true); + } + + inline MotJitContext* GenerateJitContext(JitSource* jitSource, JitPlan* jitPlan, JitContextUsage usage) + { + return GenerateFunctionJitContext( + m_function, m_procTuple, m_functionOid, m_returnSetInfo, m_queryString, jitPlan, jitSource, usage); + } + + inline JitContextType GetContextType() const + { + return JitContextType::JIT_CONTEXT_TYPE_FUNCTION; + } + +private: + PLpgSQL_function* m_function; + HeapTuple m_procTuple; + Oid m_functionOid; + ReturnSetInfo* m_returnSetInfo; + const char* m_queryString; +}; + +extern MotJitContext* JitCodegenFunction(PLpgSQL_function* function, HeapTuple procTuple, Oid functionOid, + ReturnSetInfo* returnSetInfo, JitPlan* jitPlan, JitContextUsage usage) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile MotJitContext* result = nullptr; + volatile bool nsPushed = false; + MOT::mot_string qualifiedFunctionName; + volatile const char* qualifiedFunctionNameStr = ""; + char* functionName = ""; + + u_sess->mot_cxt.jit_codegen_error = 0; + + PG_TRY(); + { + bool procNameIsNull = false; + Datum procNameDatum = SysCacheGetAttr(PROCOID, procTuple, Anum_pg_proc_proname, &procNameIsNull); + if (procNameIsNull) { + MOT_LOG_TRACE( + "Stored procedure is not jittable: catalog entry for stored procedure contains null attributes"); + } else { + // generate function context sub-queries under name-space + functionName = NameStr(*DatumGetName(procNameDatum)); + if (!qualifiedFunctionName.format("%s.%u", functionName, (unsigned)functionOid)) { + MOT_LOG_TRACE("Failed to format qualified function name"); + } else { + qualifiedFunctionNameStr = qualifiedFunctionName.c_str(); + if (PushJitSourceNamespace(functionOid, (const char*)qualifiedFunctionNameStr)) { + nsPushed = true; + MOT_LOG_TRACE("Generating code for function: %s", qualifiedFunctionNameStr); + uint64_t startTime = GetSysClock(); + JitFunctionGenerator generator( + function, procTuple, functionOid, returnSetInfo, (const char*)qualifiedFunctionNameStr); + result = JitCodegen((const char*)qualifiedFunctionNameStr, jitPlan, generator, true, usage); + if (result != nullptr) { + uint64_t endTime = GetSysClock(); + uint64_t timeMicros = MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime); + MOT_LOG_TRACE("Code generation time %" PRIu64 " micros for function: %s", + timeMicros, + qualifiedFunctionNameStr); + } + } + } + } + } + PG_CATCH(); + { + // cleanup + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while generating code for stored procedure %s: %s", + qualifiedFunctionNameStr, + edata->message); + ereport(WARNING, + (errmsg("Failed to generate code for stored procedure '%s' for MOT jitted execution.", + qualifiedFunctionNameStr), + errdetail("%s", edata->detail))); + u_sess->mot_cxt.jit_codegen_error = edata->sqlerrcode; + FreeErrorData(edata); + FlushErrorState(); + } + PG_END_TRY(); + + if (nsPushed) { + PopJitSourceNamespace(); + } + + if (result == nullptr) { + MOT_LOG_WARN("Failed to JIT-compile stored procedure: %s", qualifiedFunctionNameStr); + } + return (MotJitContext*)result; +} + +extern void JitResetScan(MotJitContext* jitContext) +{ + // ATTENTION: due to order of events in exec_bind_message() (JitResetScan is called before TryRevalidateJitContext) + // it is possible that the current JIT context is in temporary compile state, but is about to be undergo successful + // revalidation, so we do not want to miss out the reset-scan operation in this case. + // Therefore, we allow reset-scan for any context with valid execution state. + MOT_LOG_DEBUG("JitResetScan(): Resetting iteration count for context %p", jitContext); + MOT_ASSERT(jitContext->m_contextType != JitContextType::JIT_CONTEXT_TYPE_INVALID); + if (jitContext->m_execState != nullptr) { + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + JitQueryExecState* execState = (JitQueryExecState*)jitContext->m_execState; + if (execState != nullptr) { + execState->m_iterCount = 0; + } + } else if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + JitFunctionExecState* execState = (JitFunctionExecState*)jitContext->m_execState; + if (execState != nullptr) { + if (execState->m_currentSubQueryId >= 0) { + JitFunctionContext* funcContext = (JitFunctionContext*)jitContext; + MotJitContext* activeQueryContext = + funcContext->m_SPSubQueryList[execState->m_currentSubQueryId].m_queryContext; + if (activeQueryContext != nullptr) { + JitResetScan(activeQueryContext); + } + } + } + } + } +} + +static MOT::RC ValidatePrepare(MotJitContext* jitContext) +{ // make sure we can execute (guard against crash due to logical error) -#ifdef MOT_JIT_DEBUG - MOT_LOG_DEBUG("Executing JIT context %p with query: %s", jitContext, jitContext->m_queryString); -#endif - if (!jitContext->m_llvmFunction && !jitContext->m_tvmFunction) { + bool missingFunction = false; + if (!jitContext->m_llvmFunction && !jitContext->m_llvmSPFunction) { + missingFunction = true; + } + if (missingFunction) { MOT_REPORT_ERROR(MOT_ERROR_INVALID_ARG, "Execute JIT", "Cannot execute jitted function: function is null. Aborting transaction."); JitStatisticsProvider::GetInstance().AddFailExecQuery(); - report_pg_error(MOT::RC_ERROR); // execution control ends, calls ereport(error,...) + if (IsJitSubContextInline(jitContext)) { + return MOT::RC_ERROR; + } + report_pg_error(MOT::RC_ERROR, NULL); // execution control ends, calls ereport(error,...) } - // when running under thread-pool, it is possible to be executed from a thread that hasn't yet executed any MOT code - // and thus lacking a thread id and numa node id. Nevertheless, we can be sure that a proper session context is set - // up. - EnsureSafeThreadAccess(); + if (!IsJitSubContextInline(jitContext)) { + // Ensure that MOT FDW routine and Xact callbacks are registered. + if (!u_sess->mot_cxt.callbacks_set) { + ForeignDataWrapper* fdw = GetForeignDataWrapperByName(MOT_FDW, false); + if (fdw != NULL) { + (void)GetFdwRoutine(fdw->fdwhandler); + } + } + + // Ensure all MOT thread/session-local identifiers are in place. + PrepareSessionAccess(); + } // make sure we have a valid transaction object (independent of MOT FDW) u_sess->mot_cxt.jit_txn = u_sess->mot_cxt.txn_manager; - if (u_sess->mot_cxt.jit_txn == NULL) { + if (u_sess->mot_cxt.jit_txn == nullptr) { MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "Execute JIT", "Cannot execute jitted code: Current transaction is undefined. Aborting transaction."); JitStatisticsProvider::GetInstance().AddFailExecQuery(); - report_pg_error(MOT::RC_MEMORY_ALLOCATION_ERROR); // execution control ends, calls ereport(error,...) + if (IsJitSubContextInline(jitContext)) { + return MOT::RC_MEMORY_ALLOCATION_ERROR; + } + report_pg_error(MOT::RC_MEMORY_ALLOCATION_ERROR, NULL); // execution control ends, calls ereport + } + + if (u_sess->mot_cxt.jit_txn->IsTxnAborted()) { + raiseAbortTxnError(); + } + + return MOT::RC_OK; +} + +static int JitPrepareExec(MotJitContext* jitContext) +{ + if (!IsJitSubContextInline(jitContext)) { + MOT_LOG_TRACE("Executing JIT context %p with query/function: %s", jitContext, jitContext->m_queryString); + } else { + MOT_LOG_DEBUG("Executing JIT context %p with query/function: %s", jitContext, jitContext->m_queryString); + } + + MOT::RC result = ValidatePrepare(jitContext); + if (result != MOT::RC_OK) { + return result; } // during the very first invocation of the query we need to setup the reusable search keys // This is also true after TRUNCATE TABLE, in which case we also need to re-fetch all index objects - if ((jitContext->m_argIsNull == nullptr) || - ((jitContext->m_commandType != JIT_COMMAND_INSERT) && (jitContext->m_index == nullptr))) { + bool needPrepare = ((jitContext->m_execState == nullptr) || MOT_ATOMIC_LOAD(jitContext->m_execState->m_purged)); + if (needPrepare) { + u_sess->mot_cxt.jit_codegen_error = 0; if (!PrepareJitContext(jitContext)) { + int errorCode = MOT_ERROR_OOM; + if (u_sess->mot_cxt.jit_codegen_error == ERRCODE_QUERY_CANCELED) { + errorCode = MOT_ERROR_STATEMENT_CANCELED; + } else { + int rootErrorCode = MOT::GetRootError(); + if (rootErrorCode != MOT_NO_ERROR) { + MOT::RC rootRC = MOT::ErrorToRC(rootErrorCode); + MOT_LOG_TRACE("Root error is %d %s, translated to %d %s", + rootErrorCode, + MOT::ErrorCodeToString(rootErrorCode), + rootRC, + MOT::RcToString(rootRC)); + errorCode = rootErrorCode; + } + } + result = MOT::ErrorToRC(errorCode); MOT_REPORT_ERROR( - MOT_ERROR_OOM, "Execute JIT", "Failed to prepare for executing jitted code, aborting transaction"); + errorCode, "Execute JIT", "Failed to prepare for executing jitted code, aborting transaction"); JitStatisticsProvider::GetInstance().AddFailExecQuery(); - report_pg_error(MOT::RC_MEMORY_ALLOCATION_ERROR); // execution control ends, calls ereport(error,...) + if (IsJitSubContextInline(jitContext)) { + return result; + } + report_pg_error(result, NULL); // execution control ends, calls ereport } } - // setup current JIT context - u_sess->mot_cxt.jit_context = jitContext; + if (!IsJitSubContextInline(jitContext)) { + u_sess->mot_cxt.jit_txn->SetIsolationLevel(u_sess->utils_cxt.XactIsoLevel); + } + + // each query should start with m_rc = OK to avoid reporting wrong query result due to previous run + jitContext->m_rc = MOT::RC_OK; + + return MOT::RC_OK; +} #ifdef MOT_JIT_DEBUG - // in trace log-level we raise the log level to DEBUG on first few executions only - bool firstExec = false; - if ((++jitContext->m_execCount <= 2) && MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { - MOT_LOG_TRACE("Executing JIT context %p (exec: %" PRIu64 ", query: %" PRIu64 ", iteration: %" PRIu64 "): %s", +#define MOT_JIT_DEBUG_EXEC_COUNT 2 + +static bool DebugPrintQueryExecStats(MotJitContext* jitContext, JitQueryExecState* execState) +{ + bool debugExec = false; + if ((++execState->m_execCount <= MOT_JIT_DEBUG_EXEC_COUNT) && MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { + MOT_LOG_TRACE("Executing JIT query context %p (exec: %" PRIu64 ", query: %" PRIu64 ", iteration: %" PRIu64 + "): %s", jitContext, - jitContext->m_execCount, - jitContext->m_queryCount, - jitContext->m_iterCount, + execState->m_execCount, + execState->m_queryCount, + execState->m_iterCount, jitContext->m_queryString); - MOT::SetLogComponentLogLevel("JitExec", MOT::LogLevel::LL_DEBUG); - firstExec = true; + debugExec = true; } + return debugExec; +} + +static bool DebugPrintFunctionExecStats(MotJitContext* jitContext, JitFunctionExecState* execState) +{ + bool debugExec = false; + if ((++execState->m_execCount <= MOT_JIT_DEBUG_EXEC_COUNT) && MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { + MOT_LOG_TRACE("Executing JIT function context %p (exec: %" PRIu64 "): %s", + jitContext, + execState->m_execCount, + jitContext->m_queryString); + debugExec = true; + } + return debugExec; +} #endif - // update iteration count and identify a new scan - int newScan = 0; - if (jitContext->m_iterCount == 0) { - ++jitContext->m_queryCount; - newScan = 1; -#ifdef MOT_JIT_DEBUG - MOT_LOG_TRACE("Starting a new scan (exec: %" PRIu64 ", query: %" PRIu64 ",iteration: %" PRIu64 ") for query %s", - jitContext->m_execCount, - jitContext->m_queryCount, - jitContext->m_iterCount, - jitContext->m_queryString); -#endif - } - ++jitContext->m_iterCount; - - // invoke the jitted function - if (jitContext->m_llvmFunction != nullptr) { -#ifdef MOT_JIT_DEBUG - MOT_LOG_DEBUG("Executing LLVM-jitted function %p: %s", jitContext->m_llvmFunction, jitContext->m_queryString); -#endif - result = ((JitFunc)jitContext->m_llvmFunction)(jitContext->m_table, - jitContext->m_index, - jitContext->m_searchKey, - jitContext->m_bitmapSet, - params, - slot, - tuplesProcessed, - scanEnded, - newScan, - jitContext->m_endIteratorKey, - jitContext->m_innerTable, - jitContext->m_innerIndex, - jitContext->m_innerSearchKey, - jitContext->m_innerEndIteratorKey); +#ifdef MOT_JIT_TEST +static void UpdateTestStats(MotJitContext* jitContext, int newScan) +{ + if (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY) { + if (newScan) { + MOT_ATOMIC_INC(totalQueryExecCount); + MOT_LOG_DEBUG("Updated query count to %" PRIu64 ": %s", totalQueryExecCount, jitContext->m_queryString); + } else { + MOT_LOG_DEBUG("Skipped updated query count, not a new scan: %s", jitContext->m_queryString); + } } else { -#ifdef MOT_JIT_DEBUG - MOT_LOG_DEBUG("Executing TVM-jitted function %p: %s", jitContext->m_tvmFunction, jitContext->m_queryString); -#endif - result = JitExecTvmQuery(jitContext, params, slot, tuplesProcessed, scanEnded, newScan); + MOT_ATOMIC_INC(totalFunctionExecCount); + MOT_LOG_DEBUG("Updated function count to %" PRIu64 ": %s", totalFunctionExecCount, jitContext->m_queryString); } +} +#endif #ifdef MOT_JIT_DEBUG - if (firstExec) { - MOT::SetLogComponentLogLevel("JitExec", MOT::LogLevel::LL_TRACE); +static void JitExecWrapUp(MotJitContext* jitContext, MotJitContext* prevContext, int result, int newScan, + bool debugExec, MOT::LogLevel prevLevel) +#else +static void JitExecWrapUp(MotJitContext* jitContext, MotJitContext* prevContext, int result, int newScan) +#endif +{ +#ifdef MOT_JIT_DEBUG + if (debugExec && !IsJitSubContextInline(jitContext)) { + (void)MOT::SetLogComponentLogLevel("JitExec", prevLevel); } #endif - // reset current JIT context - u_sess->mot_cxt.jit_context = NULL; + // restore previous JIT context + if (prevContext) { + // pull up error information + prevContext->m_execState->m_nullColumnId = u_sess->mot_cxt.jit_context->m_execState->m_nullColumnId; + prevContext->m_execState->m_nullViolationTable = u_sess->mot_cxt.jit_context->m_execState->m_nullViolationTable; + } + u_sess->mot_cxt.jit_context = prevContext; + + if (result == 0 && jitContext->m_rc != MOT::RC_OK) { + // in case query result is OK, we need to verify that no error was raised while processing it. + MOT_LOG_TRACE( + "JitExecWrapUp(): setting result to (%s), jitContext->m_rc = %u, result = %u, jitContext = %p, query: (%s)", + RcToString(jitContext->m_rc), + jitContext->m_rc, + result, + jitContext, + jitContext->m_queryString); + result = jitContext->m_rc; + jitContext->m_rc = MOT::RC_OK; + } if (result == 0) { + // update statistics JitStatisticsProvider::GetInstance().AddInvokeQuery(); - if (newScan) { #ifdef MOT_JIT_TEST - MOT_ATOMIC_INC(totalExecCount); - MOT_LOG_INFO("JIT total queries executed: %" PRIu64, MOT_ATOMIC_LOAD(totalExecCount)); + UpdateTestStats(jitContext, newScan); #endif - JitStatisticsProvider::GetInstance().AddExecQuery(); - } + JitStatisticsProvider::GetInstance().AddExecQuery(); } else { ProcessJitResult((MOT::RC)result, jitContext, newScan); } +} + +#ifdef MOT_JIT_DEBUG +static void DebugPrintParams(ParamListInfo params, JitCommandType commandType) +{ + int paramCount = params ? params->numParams : 0; + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { + MOT_LOG_DEBUG("Executing %s command with %d parameters: ", CommandToString(commandType), paramCount); + for (int i = 0; i < paramCount; ++i) { + if (!params->params[i].isnull) { + MOT_LOG_BEGIN(MOT::LogLevel::LL_DEBUG, "Param %d: ", i); + PrintDatum(MOT::LogLevel::LL_DEBUG, + params->params[i].ptype, + params->params[i].value, + params->params[i].isnull); + MOT_LOG_END(MOT::LogLevel::LL_DEBUG); + } + } + } +} +#define DEBUG_PRINT_PARAMS(params, commandType) DebugPrintParams(params, commandType) +#else +#define DEBUG_PRINT_PARAMS(params, commandType) +#endif + +static inline int JitExecGetNextTuple(JitQueryContext* jitContext, ParamListInfo params, TupleTableSlot* slot, + uint64_t* tuplesProcessed, int* scanEnded, int newScan) +{ + int result = MOT::RC_OK; + + if (jitContext->m_llvmFunction != nullptr) { +#ifdef MOT_JIT_DEBUG + MOT_LOG_DEBUG("JitExecGetNextTuple(): Executing LLVM-jitted function %p: %s", + jitContext->m_llvmFunction, + jitContext->m_queryString); +#endif + result = JitExecLlvmQuery(jitContext, params, slot, tuplesProcessed, scanEnded, newScan); + } else { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("Cannot execute LLVM function, function is missing: %s", jitContext->m_queryString))); + } return result; } -extern void PurgeJitSourceCache(uint64_t relationId, bool purgeOnly) +static inline int JitExecCreateNonNativeTupleSort(MotJitContext* jitContext, + JitNonNativeSortExecState* jitNonNativeSortExecState, ParamListInfo params, TupleTableSlot* slot) { - MOT_LOG_TRACE("Purging JIT source map by relation id %" PRIu64, relationId); - (void)PurgeJitSourceMap(relationId, purgeOnly); + JitQueryContext* queryContext = (JitQueryContext*)jitContext; + JitNonNativeSortParams* sortParams = queryContext->m_nonNativeSortParams; + + int64 sortMem = SET_NODEMEM(0, 0); + int64 maxMem = 0; // No limit + + // Iterator's direction is consistent over single query + bool randomAccess = false; + + // Slot for my result tuples + TupleDesc tupDesc = slot->tts_tupleDescriptor; + + if (!tupDesc) { + MOT_LOG_TRACE("JitExecCreateNonNativeTupleSort(): Tuple descriptor (from slot) is NULL. slot: (%p)", slot); + MOT_LOG_ERROR("Tuple descriptor is invalid"); + + return MOT::RC_ERROR; + } + + // The parallel num of plan. Not relvant for MOT as this stage + int dop = SET_DOP(0); + + // Session memory context is used (instead of query context) as the allocated tupleSortState might be released only + // when the jit context is destroyed (In case the user didnt pull all results. e.g. In batch mode, the user can stop + // the query in the middle). In this case, when using query memory context, we will try to free\destroy a memory + // that was already freed + MemoryContext oldCxt = MemoryContextSwitchTo(SESS_GET_MEM_CXT_GROUP(MEMORY_CONTEXT_EXECUTOR)); + Tuplesortstate* tupleSortState = tuplesort_begin_heap(tupDesc, + sortParams->numCols, + sortParams->sortColIdx, + sortParams->sortOperators, + sortParams->collations, + sortParams->nullsFirst, + sortMem, + randomAccess, + maxMem, + sortParams->plan_node_id, + dop); + + (void)MemoryContextSwitchTo(oldCxt); + if (sortParams->bound) { + tuplesort_set_bound(tupleSortState, sortParams->bound); + } + + // local variables for retrieving all tuples + int scanEnded = 0; + uint64_t tuplesProcessed = 0; + int newScan = 1; + int result = MOT::RC_OK; + uint64_t numRetrievedTuples = 0; // For debug only + + // Scan the index and feed all the tuples to tuplesort + while (true) { + tuplesProcessed = 0; + result = JitExecGetNextTuple(queryContext, params, slot, &tuplesProcessed, &scanEnded, newScan); + + if (scanEnded) { + ExecClearTuple(slot); + MOT_LOG_DEBUG( + "JitExecCreateNonNativeTupleSort(): Scan has ended. Total tuples received: %lu", numRetrievedTuples); + break; + } + + if (result != MOT::RC_OK || tuplesProcessed == 0) { + MOT_LOG_TRACE("JitExecCreateNonNativeTupleSort(): Failed to retrieve all tuples. Tuples retrieved: %lu, " + "l_tuplesProcessed: %lu, jitContext->m_llvmFunction: %p, query: %s", + numRetrievedTuples, + tuplesProcessed, + jitContext->m_llvmFunction, + jitContext->m_queryString); + MOT_LOG_ERROR("Failed to retrieve all tuples. Tuples retrieved: %lu, query: %s", + numRetrievedTuples, + (jitContext->m_queryString ? jitContext->m_queryString : "null")); + + ExecClearTuple(slot); + tuplesort_end(tupleSortState); + tupleSortState = nullptr; + // this is an error state. MOT::RC_LOCAL_ROW_NOT_FOUND might not interpreted as error, so we need to + // replace it with MOT::RC_ERROR. + return (result == MOT::RC_LOCAL_ROW_NOT_FOUND ? MOT::RC_ERROR : result); + } + + // if PGXC is defined, tuplesort_puttupleslotontape is used in some case (instead of tuplesort_puttupleslot). + // We will need to address this issue in the future. + tuplesort_puttupleslot(tupleSortState, slot); + numRetrievedTuples++; + newScan = 0; + } + + sort_count(tupleSortState); + + // Complete the sort + tuplesort_performsort(tupleSortState); + + jitNonNativeSortExecState->m_tupleSort = tupleSortState; + + return MOT::RC_OK; +} + +static inline bool JitExecOrderedQueryIsLimitReached( + JitNonNativeSortParams* sortParams, JitQueryExecState* jitQueryExecState) +{ + if (sortParams->bound && jitQueryExecState->m_limitCounter >= (uint32_t)sortParams->bound) { + return true; + } + + return false; +} + +static int JitExecOrderedQuery(MotJitContext* jitContext, JitQueryExecState* jitQueryExecState, ParamListInfo params, + TupleTableSlot* slot, uint64_t* tuplesProcessed, int* scanEnded, int newScan) +{ + JitNonNativeSortExecState* jitNonNativeSortExecState = jitQueryExecState->m_nonNativeSortExecState; + JitQueryContext* queryContext = (JitQueryContext*)jitContext; + JitNonNativeSortParams* sortParams = queryContext->m_nonNativeSortParams; + + MOT_ASSERT(jitNonNativeSortExecState); + + if (newScan) { + MOT_LOG_TRACE("JitExecOrderedQuery(): Initializing JitNonNativeSortExecState for new sort for query: (%s)", + jitContext->m_queryString); + + // If tupleSortState already exists, we need to re-init the sort state + if (jitNonNativeSortExecState->m_tupleSort) { + tuplesort_end(jitNonNativeSortExecState->m_tupleSort); + jitNonNativeSortExecState->m_tupleSort = nullptr; + } + + int result = JitExecCreateNonNativeTupleSort(jitContext, jitNonNativeSortExecState, params, slot); + if (result != MOT::RC_OK) { + MOT_LOG_ERROR("Failed to create Tuple sort state for non native sort"); + return result; + } + + jitQueryExecState->m_limitCounter = 0; + } + + if (!jitNonNativeSortExecState->m_tupleSort) { + // This is not a new scan (newScan == 0) and we dont have a tupleSortState probably because we already destroyed + // it after setting *scanEnded = 1 + MOT_LOG_ERROR("Tuple was requested after scanEnded flag was raised. newScan %d", newScan); + *scanEnded = 1; + *tuplesProcessed = 0; + return MOT::RC_LOCAL_ROW_NOT_FOUND; + } + + // If limit clause, enforce number of returned tuples before get next tuple from tuplesort. + // Update scanEnded if last tuple was already provided + ExecClearTuple(slot); + if (JitExecOrderedQueryIsLimitReached(sortParams, jitQueryExecState) || + !(tuplesort_gettupleslot( + jitNonNativeSortExecState->m_tupleSort, ScanDirectionIsForward(sortParams->scanDir), slot, NULL))) { + MOT_LOG_TRACE("JitExecOrderedQuery(): limit reached or tuple sort is empty"); + tuplesort_end(jitNonNativeSortExecState->m_tupleSort); + jitNonNativeSortExecState->m_tupleSort = nullptr; + *scanEnded = 1; + *tuplesProcessed = 0; + return MOT::RC_LOCAL_ROW_NOT_FOUND; + } + + MOT_LOG_DEBUG("JitExecOrderedQuery(): returning next tuple from tuple sort"); + *tuplesProcessed = 1; + *scanEnded = 0; + if (sortParams->bound) { + // Increase tuple counter to support limit clause + jitQueryExecState->m_limitCounter++; + } + + return MOT::RC_OK; +} + +extern int JitExecQuery( + MotJitContext* jitContext, ParamListInfo params, TupleTableSlot* slot, uint64_t* tuplesProcessed, int* scanEnded) +{ + MOT_ASSERT(jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY); + + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitQueryContext* queryContext = (JitQueryContext*)jitContext; + + volatile int result = JitPrepareExec(jitContext); + if (result != MOT::RC_OK) { + return result; + } + + // setup current JIT context + volatile MotJitContext* prevContext = u_sess->mot_cxt.jit_context; + u_sess->mot_cxt.jit_context = jitContext; + + // we avoid weird stuff by putting null in case of internal error + bool isQueryContext = (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY); + JitQueryExecState* execState = isQueryContext ? (JitQueryExecState*)jitContext->m_execState : nullptr; + MOT_ASSERT(execState); +#ifdef MOT_JIT_DEBUG + // in trace log-level we raise the log level to DEBUG on first few executions only + volatile MOT::LogLevel prevLevel = MOT::LogLevel::LL_INFO; + volatile bool debugExec = DebugPrintQueryExecStats(jitContext, execState); + if (debugExec) { + prevLevel = MOT::SetLogComponentLogLevel("JitExec", MOT::LogLevel::LL_DEBUG); + } +#endif + + // update iteration count and identify a new scan + volatile int newScan = 0; + if (execState->m_iterCount == 0) { + ++execState->m_queryCount; + newScan = 1; +#ifdef MOT_JIT_DEBUG + MOT_LOG_TRACE("Starting a new scan (exec: %" PRIu64 ", query: %" PRIu64 ", iter: %" PRIu64 ") for query %s", + execState->m_execCount, + execState->m_queryCount, + execState->m_iterCount, + jitContext->m_queryString); +#endif + } + ++execState->m_iterCount; + + // prepare invoke query parameters (invoked stored procedure reports results directly into caller's variables) + if (jitContext->m_commandType == JIT_COMMAND_INVOKE) { + execState->m_directParams = params; + execState->m_invokeSlot = slot; + execState->m_invokeTuplesProcessed = tuplesProcessed; + execState->m_invokeScanEnded = scanEnded; + } + DEBUG_PRINT_PARAMS(params, jitContext->m_commandType); + + // clear error stack before new execution + if (!IsJitSubContextInline(jitContext)) { + MOT::ClearErrorStack(); + } + + // reset error state before new execution + ResetErrorState(execState); + + // invoke the jitted function + volatile MemoryContext origCxt = CurrentMemoryContext; + + // init out params + *scanEnded = 0; + *tuplesProcessed = 0; + + PG_TRY(); + { + if (queryContext->m_commandType != JIT_COMMAND_INVOKE) { + MOT_ASSERT(queryContext->m_commandType != JIT_COMMAND_FUNCTION); + } + if (IsWriteCommand(queryContext->m_commandType)) { + (void)::GetCurrentTransactionId(); + } + if (queryContext->m_nonNativeSortParams) { + result = JitExecOrderedQuery(jitContext, execState, params, slot, tuplesProcessed, scanEnded, newScan); + } else { + result = + JitExecGetNextTuple((JitQueryContext*)queryContext, params, slot, tuplesProcessed, scanEnded, newScan); + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN( + "Caught exception while executing jitted query '%s': %s", queryContext->m_queryString, edata->message); + ereport(WARNING, + (errmsg("Failed to execute MOT-jitted query '%s'.", queryContext->m_queryString), + errdetail("%s", edata->detail))); + FlushErrorState(); + FreeErrorData(edata); + result = (int)MOT::RC_JIT_SP_EXCEPTION; + } + PG_END_TRY(); + +#ifdef MOT_JIT_DEBUG + JitExecWrapUp((MotJitContext*)jitContext, (MotJitContext*)prevContext, result, newScan, debugExec, prevLevel); +#else + JitExecWrapUp((MotJitContext*)jitContext, (MotJitContext*)prevContext, result, newScan); +#endif + return result; +} + +static void CopyTupleTableSlot(TupleTableSlot* tuple) +{ + // switch to memory context of caller + MemoryContext oldCtx = SwitchToSPICallerContext(); + + // copy slot values to memory context of caller + TupleDesc tupDesc = tuple->tts_tupleDescriptor; + for (int i = 0; i < tupDesc->natts; ++i) { + // skip dropped columns in destination + if (tupDesc->attrs[i].attisdropped) { + continue; + } + + bool isNull = tuple->tts_isnull[i]; + Datum value = tuple->tts_values[i]; + Oid type = tuple->tts_tupleDescriptor->attrs[i].atttypid; + + // perform proper type conversion as in exec_assign_value() at pl_exec.cpp + MOT_LOG_DEBUG("CopyTupleTableSlot(): Copying datum %d of type %u", i, type); + Datum resValue = CopyDatum(value, type, isNull); + tuple->tts_values[i] = resValue; + } + + // restore current memory context + if (oldCtx) { + (void)MemoryContextSwitchTo(oldCtx); + } +} + +extern int JitExecFunction( + MotJitContext* jitContext, ParamListInfo params, TupleTableSlot* slot, uint64_t* tuplesProcessed, int* scanEnded) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + MOT_ASSERT(jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + volatile JitFunctionContext* functionContext = (JitFunctionContext*)jitContext; + + volatile int result = JitPrepareExec(jitContext); + if (result != MOT::RC_OK) { + return result; + } + + // we avoid weird stuff by putting null in case of internal error + bool isFunctionContext = (jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + volatile JitFunctionExecState* execState = + isFunctionContext ? (JitFunctionExecState*)jitContext->m_execState : nullptr; + MOT_ASSERT(execState); + execState->m_spiBlockId = 0; // reset each time before execution +#ifdef MOT_JIT_DEBUG + // in trace log-level we raise the log level to DEBUG on first few executions only + volatile MOT::LogLevel prevLevel = MOT::LogLevel::LL_INFO; + volatile bool debugExec = DebugPrintFunctionExecStats(jitContext, (JitFunctionExecState*)execState); + if (debugExec) { + prevLevel = MOT::SetLogComponentLogLevel("JitExec", MOT::LogLevel::LL_DEBUG); + } +#endif + + // save parameters with which the function was invoked so they can later be pushed down to sub-queries + execState->m_functionParams = params; + + // reset error state + ResetErrorState((JitExec::JitExecState*)execState); + execState->m_exceptionOrigin = JIT_EXCEPTION_INTERNAL; + + // clear error stack before new execution + if (!IsJitSubContextInline(jitContext)) { + MOT::ClearErrorStack(); + } + DEBUG_PRINT_PARAMS(params, jitContext->m_commandType); + + // connect to SPI + SPIAutoConnect spiAutoConn(false, functionContext->m_functionOid); + if (!spiAutoConn.IsConnected()) { + int rc = spiAutoConn.GetErrorCode(); + MOT_LOG_ERROR("SPI_connect failed with error code %d: %s when executing MOT jitted function %s.", + rc, + SPI_result_code_string(rc), + jitContext->m_queryString); + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("SPI_connect failed with error code %d: %s when executing MOT jitted function %s.", + rc, + SPI_result_code_string(rc), + jitContext->m_queryString))); + } + + // setup current JIT context + volatile MotJitContext* prevContext = u_sess->mot_cxt.jit_context; + u_sess->mot_cxt.jit_context = jitContext; + + // invoke the jitted function + volatile MemoryContext origCxt = CurrentMemoryContext; + // since we use parts of the PG function in query invocation, we need to make sure it does not get deleted + ++execState->m_function->use_count; + MOT_LOG_DEBUG("JitExecFunction(): Increased use count of function %p to %lu: %s", + execState->m_function, + execState->m_function->use_count, + jitContext->m_queryString); + PG_TRY(); + { + if (jitContext->m_llvmSPFunction != nullptr) { +#ifdef MOT_JIT_DEBUG + MOT_LOG_DEBUG( + "Executing LLVM-jitted function %p: %s", jitContext->m_llvmSPFunction, jitContext->m_queryString); +#endif + result = + JitExecLlvmFunction((JitFunctionContext*)functionContext, params, slot, tuplesProcessed, scanEnded); + } else { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("Cannot execute LLVM function, function is missing: %s", jitContext->m_queryString))); + } + + if (result == MOT::RC_OK) { + // copy tuple to caller's memory context + CopyTupleTableSlot(slot); + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while executing jitted function '%s': %s", + functionContext->m_queryString, + edata->message); + ereport(WARNING, + (errmodule(MOD_MOT), + errmsg("Failed to execute MOT-jitted stored procedure '%s'.", functionContext->m_queryString), + errdetail("%s", edata->detail))); + FlushErrorState(); + FreeErrorData(edata); + + // clean up: roll back all open sub-transactions, and open SPI connections + JitReleaseAllSubTransactions(true, InvalidSubTransactionId); + while (execState->m_spiBlockId > 0) { + JitCleanupBlockAfterException(); + } + result = (int)MOT::RC_ERROR; + } + PG_END_TRY(); + --execState->m_function->use_count; + MOT_LOG_DEBUG("JitExecFunction(): Decreased use count of function %p to %lu: %s", + execState->m_function, + execState->m_function->use_count, + jitContext->m_queryString); + + // disconnect from SPI - this includes handling any error and intermediate SPI state + spiAutoConn.Disconnect(); + +#ifdef MOT_JIT_DEBUG + JitExecWrapUp((MotJitContext*)jitContext, (MotJitContext*)prevContext, result, 1, debugExec, prevLevel); +#else + JitExecWrapUp((MotJitContext*)jitContext, (MotJitContext*)prevContext, result, 1); +#endif + return result; +} + +extern void PurgeJitSourceCache( + uint64_t objectId, JitPurgeScope purgeScope, JitPurgeAction purgeAction, const char* funcName) +{ + // since this might be called through initdb, we make a safety check here + if (MOT::MOTEngine::GetInstance() == nullptr) { + return; + } + + // ensure all MOT thread/session-local identifiers are in place + PrepareSessionAccess(); + + MOT_LOG_TRACE("Purging JIT source map by object id %" PRIu64 " scope: %s, action: %s", + objectId, + JitPurgeScopeToString(purgeScope), + JitPurgeActionToString(purgeAction)); + PurgeJitSourceMap(objectId, purgeScope, purgeAction, funcName); +} + +extern MotJitDetail* MOTGetJitDetail(uint32_t* num) +{ + *num = 0; + if (!IsMotCodegenEnabled()) { + return nullptr; + } + + // ensure all MOT thread/session-local identifiers are in place + PrepareSessionAccess(); + + MOT_LOG_DEBUG("Getting mot_jit_detail() information"); + return GetJitSourceDetail(num); +} + +extern MotJitProfile* MOTGetJitProfile(uint32_t* num) +{ + *num = 0; + if (!IsMotCodegenEnabled() || !MOT::GetGlobalConfiguration().m_enableCodegenProfile) { + return nullptr; + } + + // ensure all MOT thread/session-local identifiers are in place + PrepareSessionAccess(); + + MOT_LOG_DEBUG("Getting mot_jit_profile() information"); + return JitProfiler::GetInstance()->GetProfileReport(num); +} + +class JitQueryRegenerator : public JitFunctionQueryVisitor { +public: + JitQueryRegenerator( + PLpgSQL_function* function, JitFunctionContext* functionContext, FuncParamInfoList* paramInfoList) + : m_function(function), m_functionContext(functionContext), m_paramInfoList(paramInfoList) + {} + + ~JitQueryRegenerator() final + {} + + JitVisitResult OnQuery(PLpgSQL_expr* expr, PLpgSQL_row* row, int index, bool into) final + { + // 1. find corresponding sub-query context + // 2. check if it is invalid + // 3. regenerate code if required + JitCallSite* callSite = nullptr; + uint32_t subQueryIndex = (uint32_t)-1; + for (uint32_t i = 0; i < m_functionContext->m_SPSubQueryCount; ++i) { + if (m_functionContext->m_SPSubQueryList[i].m_exprIndex == index) { + callSite = &m_functionContext->m_SPSubQueryList[i]; + subQueryIndex = i; + MOT_LOG_TRACE( + "Found expression in sub-query %u/%u: %s", i, m_functionContext->m_SPSubQueryCount, expr->query); + break; + } + } + if (callSite == nullptr) { + MOT_LOG_TRACE("Expression %d at %p to be regenerated not found: %s", index, expr, expr->query); + return JitVisitResult::JIT_VISIT_CONTINUE; // go on, that's fine + } + MOT_ASSERT(subQueryIndex != (uint32_t)-1); + MotJitContext* subQueryContext = callSite->m_queryContext; + if (subQueryContext == nullptr) { + if (strcmp(expr->query, callSite->m_queryString) != 0) { + MOT_LOG_TRACE("Mismatching non-jittable sub-query %u: %s", subQueryIndex, expr->query); + return JitVisitResult::JIT_VISIT_ERROR; + } + return JitVisitResult::JIT_VISIT_CONTINUE; + } + if ((subQueryContext->m_queryString != nullptr) && (strcmp(expr->query, subQueryContext->m_queryString) != 0)) { + MOT_LOG_TRACE("Mismatching jittable sub-query %u: %s", subQueryIndex, expr->query); + return JitVisitResult::JIT_VISIT_ERROR; + } + uint8_t validState = MOT_ATOMIC_LOAD(subQueryContext->m_validState); + if (validState == JIT_CONTEXT_VALID) { + return JitVisitResult::JIT_VISIT_CONTINUE; + } else if (validState == JIT_CONTEXT_PENDING_COMPILE) { + MOT_LOG_TRACE("Encountered dummy sub-query %u pending for compilation: %s", subQueryIndex, expr->query); + return JitVisitResult::JIT_VISIT_CONTINUE; + } + + // in either one of these cases we regenerate code for query: + // 1. if query itself was invalidated + // 2. if this is an invoke query with directly invalid sub-SP + bool regenCode = false; + if (validState & JIT_CONTEXT_INVALID) { + MOT_LOG_TRACE("Regenerating invalid sub-query: %s", subQueryContext->m_queryString); + regenCode = true; + } else if (validState & JIT_CONTEXT_DEPRECATE) { + MOT_LOG_TRACE("Regenerating deprecate sub-query: %s", subQueryContext->m_queryString); + regenCode = true; + } else if (validState & JIT_CONTEXT_CHILD_SP_INVALID) { + MOT_LOG_TRACE("Inspecting sub-query with invalid sub-SP: %s", subQueryContext->m_queryString); + MOT_ASSERT(subQueryContext->m_commandType == JIT_COMMAND_INVOKE); + MotJitContext* invokedContext = ((JitQueryContext*)subQueryContext)->m_invokeContext; + if (invokedContext == nullptr) { + MOT_LOG_TRACE("Regenerating sub-query with null sub-SP: %s", subQueryContext->m_queryString); + regenCode = true; + } else if (MOT_ATOMIC_LOAD(invokedContext->m_validState) & JIT_CONTEXT_INVALID) { + MOT_LOG_TRACE("Regenerating sub-query with invalid sub-SP: %s", subQueryContext->m_queryString); + regenCode = true; + } + } + if (regenCode) { + // we now generate code as usual + MOT_LOG_TRACE("Regenerating code for invalid sub-query: %s", expr->query); + if (!RegenerateStmtQuery(expr, + row, + m_paramInfoList, + m_function, + callSite, + m_functionContext->m_SPSubQueryCount, + subQueryIndex)) { + MOT_LOG_TRACE("Failed to regenerate sub-query: %s", expr->query); + return JitVisitResult::JIT_VISIT_ERROR; + } + // reinstate parent context (careful, this might be a non-jittable sub-query) + if (callSite->m_queryContext != nullptr) { + callSite->m_queryContext->m_parentContext = m_functionContext; + } + // NOTE: JIT Source will be updated upon first access (during PREPARE) + } else { + // this must be an invoke context and the invoked child SP has some sub-SP/query that is invalid, + // so we just revalidate it + MOT_LOG_TRACE("Re-validating code for sub-query %u with invalid sub-SP: %s", subQueryIndex, expr->query); + if (!RevalidateJitContext(subQueryContext)) { + MOT_LOG_TRACE("Failed to revalidate sub-query %u: %s", subQueryIndex, expr->query); + return JitVisitResult::JIT_VISIT_ERROR; + } + } + + return JitVisitResult::JIT_VISIT_CONTINUE; + } + + void OnError(const char* stmtType, int lineNo) final + { + MOT_LOG_TRACE("Failed to regenerate SP query plans, during statement '%s' at line %d", stmtType, lineNo); + } + +private: + PLpgSQL_function* m_function; + JitFunctionContext* m_functionContext; + FuncParamInfoList* m_paramInfoList; +}; + +extern bool JitReCodegenFunctionQueries(MotJitContext* jitContext) +{ + // 1. get PG compiled function object + // 2. traverse all sub-queries + // 3. match the corresponding sub-query context + // 4. regenerate code if marked as invalid (as in first time compile) + // 5. invalid child SPs are wrapped with query INVOKE context, so they are handled transparently + + // connect to SPI outside try-catch block (since it uses longjmp - no destructors called) + SPIAutoConnect spiAutoConnect(true); + + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitFunctionContext* functionContext = (JitFunctionContext*)jitContext; + volatile FuncParamInfoList paramInfoList = {}; + enum class CodeRegenState { CRS_INIT, CRS_FUNC_PARAM, CRS_NS_PUSH, CRS_NS_POP }; + volatile CodeRegenState regenState = CodeRegenState::CRS_INIT; + volatile bool result = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile PLpgSQL_function* function = nullptr; + PG_TRY(); + { + function = GetPGCompiledFunction(functionContext->m_functionOid); + if (function == nullptr) { + MOT_LOG_TRACE( + "Cannot regenerate code for function: Failed to get function by id %u", functionContext->m_functionOid); + result = false; + } else { + // prepare execution state + ++function->use_count; + MOT_LOG_TRACE("JitReCodegenFunctionQueries(): Increased use count of function %p to %lu: %s", + function, + function->use_count, + jitContext->m_queryString); + + if (spiAutoConnect.Connect()) { + // prepare function parameters for query parsing + if (!PrepareFuncParamInfoList((PLpgSQL_function*)function, (FuncParamInfoList*)¶mInfoList)) { + MOT_LOG_TRACE("Cannot regenerate code for function: Failed to prepare function %u parameters", + functionContext->m_functionOid); + result = false; + } else { + regenState = CodeRegenState::CRS_FUNC_PARAM; + + // regenerate code for all sub-queries + JitQueryRegenerator codeRegen((PLpgSQL_function*)function, + (JitFunctionContext*)functionContext, + (FuncParamInfoList*)¶mInfoList); + + if (PushJitSourceNamespace(functionContext->m_functionOid, functionContext->m_queryString)) { + regenState = CodeRegenState::CRS_NS_PUSH; + result = VisitFunctionQueries((PLpgSQL_function*)function, &codeRegen); + PopJitSourceNamespace(); + regenState = CodeRegenState::CRS_NS_POP; + } + } + } else { + int rc = spiAutoConnect.GetErrorCode(); + MOT_REPORT_ERROR(MOT_ERROR_RESOURCE_UNAVAILABLE, + "JIT Compile", + "Cannot re-generate code for SP sub-queries: Failed to connect to SPI - %s (error code %d)", + SPI_result_code_string(rc), + rc); + result = false; + } + } + } + PG_CATCH(); + { + MOT_LOG_WARN("Caught exception while regenerating code for jitted function sub-queries"); + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while regenerating code for sub-queries of jitted function '%s': %s", + functionContext->m_queryString, + edata->message); + ereport(WARNING, + (errmsg("Failed to regenerate code for MOT-jitted stored procedure '%s'.", functionContext->m_queryString), + errdetail("%s", edata->detail))); + FlushErrorState(); + FreeErrorData(edata); + result = false; + } + PG_END_TRY(); + + switch (regenState) { + case CodeRegenState::CRS_NS_PUSH: + PopJitSourceNamespace(); // fall through + case CodeRegenState::CRS_FUNC_PARAM: + DestroyFuncParamInfoList((FuncParamInfoList*)¶mInfoList); + break; + case CodeRegenState::CRS_NS_POP: + default: + break; + } + if (function != nullptr) { + --function->use_count; + MOT_LOG_TRACE("JitReCodegenFunctionQueries(): Decreased use count of function %p to %lu: %s", + function, + function->use_count, + jitContext->m_queryString); + } + + if (!result) { + MOT_LOG_TRACE("Cannot regenerate code for function: Failed to regenerate code for sub-query of function %u", + functionContext->m_functionOid); + } + + return result; +} + +extern bool TryRevalidateJitContext(MotJitContext* jitContext, TransactionId functionTxnId /* = InvalidTransactionId */) +{ + // Ensure that MOT FDW routine and Xact callbacks are registered. + if (!u_sess->mot_cxt.callbacks_set) { + ForeignDataWrapper* fdw = GetForeignDataWrapperByName(MOT_FDW, false); + if (fdw != NULL) { + (void)GetFdwRoutine(fdw->fdwhandler); + } + } + + // Make sure session is ready for access + PrepareSessionAccess(); + + // make sure SPI is set-up on top level call when revalidate takes place with invoke context + if (jitContext->m_commandType == JIT_COMMAND_INVOKE) { + SPIAutoConnect spiAutoConn; + if (!spiAutoConn.IsConnected()) { + int rc = spiAutoConn.GetErrorCode(); + MOT_LOG_TRACE("Failed to connect to SPI while generating code for query: %s (%u)", + jitContext->m_queryString, + SPI_result_code_string(rc), + rc); + MarkJitContextErrorCompile(jitContext); + return false; + } + + return RevalidateJitContext(jitContext, functionTxnId); + } else { + return RevalidateJitContext(jitContext, functionTxnId); + } +} + +static void JitDDLCallback(uint64_t relationId, MOT::DDLAccessType event, MOT::TxnDDLPhase txnDdlPhase) +{ + switch (event) { + case MOT::DDL_ACCESS_DROP_TABLE: + case MOT::DDL_ACCESS_TRUNCATE_TABLE: + case MOT::DDL_ACCESS_DROP_INDEX: + case MOT::DDL_ACCESS_ADD_COLUMN: + case MOT::DDL_ACCESS_DROP_COLUMN: + case MOT::DDL_ACCESS_RENAME_COLUMN: + PurgeJitSourceCache(relationId, JIT_PURGE_SCOPE_QUERY, JIT_PURGE_EXPIRE, nullptr); + break; + + case MOT::DDL_ACCESS_CREATE_TABLE: + case MOT::DDL_ACCESS_CREATE_INDEX: + PurgeJitSourceCache(relationId, JIT_PURGE_SCOPE_QUERY, JIT_PURGE_EXPIRE, nullptr); + break; + + case MOT::DDL_ACCESS_UNKNOWN: + if (txnDdlPhase == MOT::TxnDDLPhase::TXN_DDL_PHASE_COMMIT) { + ApplyLocalJitSourceChanges(); + } else if (txnDdlPhase == MOT::TxnDDLPhase::TXN_DDL_PHASE_ROLLBACK) { + RevertLocalJitSourceChanges(); + } else if (txnDdlPhase == MOT::TxnDDLPhase::TXN_DDL_PHASE_POST_COMMIT_CLEANUP) { + PostCommitCleanupJitSources(); + } + break; + + default: + MOT_LOG_TRACE("Invalid DDL event: %d", event); + break; + } +} + +inline const char* XactEventToString(XactEvent event) +{ + switch (event) { + case XACT_EVENT_START: + return "START"; + case XACT_EVENT_COMMIT: + return "COMMIT"; + case XACT_EVENT_END_TRANSACTION: + return "END TXN"; + case XACT_EVENT_RECORD_COMMIT: + return "RECORD COMMIT"; + case XACT_EVENT_ABORT: + return "ABORT"; + case XACT_EVENT_PREPARE: + return "PREPARE"; + case XACT_EVENT_COMMIT_PREPARED: + return "COMMIT PREPARED"; + case XACT_EVENT_ROLLBACK_PREPARED: + return "ROLLBACK PREPARED"; + case XACT_EVENT_PREROLLBACK_CLEANUP: + return "PRE-ROLLBACK CLEANUP"; + case XACT_EVENT_POST_COMMIT_CLEANUP: + return "POST-COMMIT CLEANUP"; + default: + return "N/A"; + } +} + +static void JITXactCallback(XactEvent event, void* arg) +{ + if ((MOT::MOTEngine::GetInstance() != nullptr) && IsMotCodegenEnabled()) { + MOT_LOG_DEBUG("Received transaction call back: %u (%s)", (int)event, XactEventToString(event)); + if (event == XACT_EVENT_COMMIT) { + JitDDLCallback(0, MOT::DDL_ACCESS_UNKNOWN, MOT::TxnDDLPhase::TXN_DDL_PHASE_COMMIT); + } else if (event == XACT_EVENT_ABORT) { + JitDDLCallback(0, MOT::DDL_ACCESS_UNKNOWN, MOT::TxnDDLPhase::TXN_DDL_PHASE_ROLLBACK); + } else if (event == XACT_EVENT_POST_COMMIT_CLEANUP) { + JitDDLCallback(0, MOT::DDL_ACCESS_UNKNOWN, MOT::TxnDDLPhase::TXN_DDL_PHASE_POST_COMMIT_CLEANUP); + } + } } extern bool JitInitialize() @@ -496,20 +2135,18 @@ extern bool JitInitialize() } if (JitCanInitThreadCodeGen()) { - if (IsMotPseudoCodegenForced()) { - MOT_LOG_INFO("Forcing TVM on LLVM natively supported platform by user configuration"); - g_instance.mot_cxt.jitExecMode = JIT_EXEC_MODE_TVM; - } else { - PrintNativeLlvmStartupInfo(); - g_instance.mot_cxt.jitExecMode = JIT_EXEC_MODE_LLVM; - } + PrintNativeLlvmStartupInfo(); } else { - if (IsMotPseudoCodegenForced()) { - MOT_LOG_INFO("Forcing TVM on LLVM natively unsupported platform by user configuration"); - } else { - MOT_LOG_WARN("Defaulting to TVM on LLVM natively unsupported platform"); + // whatever happened, we are done and we allow db to continue + return true; + } + + if (MOT::GetGlobalConfiguration().m_enableCodegenProfile) { + MOT_LOG_INFO("MOT JIT profiling is enabled"); + if (!JitProfiler::CreateInstance()) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, "JIT Initialization", "Failed to create JIT profiler instance"); + return false; } - g_instance.mot_cxt.jitExecMode = JIT_EXEC_MODE_TVM; } enum InitState { JIT_INIT, JIT_CTX_POOL_INIT, JIT_SRC_POOL_INIT, JIT_INIT_DONE } initState = JIT_INIT; @@ -548,6 +2185,10 @@ extern bool JitInitialize() GetMotCodegenLimit()); break; } + MOT::MOTEngine::GetInstance()->SetDDLCallback(JitDDLCallback); + // when no DDL was issued, but only SP REPLACE/DROP, we are still missing commit/rollback notification, so we + // need to register a callback for that. This takes place in each session (see PrepareSessionAccess() above). + initState = JIT_INIT_DONE; } while (0); @@ -560,6 +2201,10 @@ extern bool JitInitialize() DestroyGlobalJitContextPool(); // fall through case JIT_INIT: + if (MOT::GetGlobalConfiguration().m_enableCodegenProfile) { + JitProfiler::DestroyInstance(); + } + // fall through default: break; } @@ -569,9 +2214,13 @@ extern bool JitInitialize() extern void JitDestroy() { + MOT::MOTEngine::GetInstance()->SetDDLCallback(nullptr); DestroyJitSourceMap(); DestroyJitSourcePool(); DestroyGlobalJitContextPool(); + if (MOT::GetGlobalConfiguration().m_enableCodegenProfile) { + JitProfiler::DestroyInstance(); + } } extern bool IsMotCodegenEnabled() @@ -579,9 +2228,14 @@ extern bool IsMotCodegenEnabled() return MOT::GetGlobalConfiguration().m_enableCodegen; } -extern bool IsMotPseudoCodegenForced() +extern bool IsMotQueryCodegenEnabled() { - return MOT::GetGlobalConfiguration().m_forcePseudoCodegen; + return IsMotCodegenEnabled() && MOT::GetGlobalConfiguration().m_enableQueryCodegen; +} + +extern bool IsMotSPCodegenEnabled() +{ + return IsMotCodegenEnabled() && MOT::GetGlobalConfiguration().m_enableSPCodegen; } extern bool IsMotCodegenPrintEnabled() @@ -593,4 +2247,65 @@ extern uint32_t GetMotCodegenLimit() { return MOT::GetGlobalConfiguration().m_codegenLimit; } + +extern void JitReportParseError(ErrorData* edata, const char* queryString) +{ + MOT_LOG_WARN("Encountered parse error: %s\n\tWhile parsing query: %s", edata->message, queryString); + if (u_sess->mot_cxt.jit_parse_error == 0) { + u_sess->mot_cxt.jit_parse_error = MOT_JIT_GENERIC_PARSE_ERROR; + } +} + +extern void CleanupJitSourceTxnState() +{ + if (MOT::MOTEngine::GetInstance() != nullptr) { + CleanupLocalJitSourceChanges(); + } + if (u_sess->mot_cxt.jit_xact_callback_registered) { + MOT_LOG_DEBUG("Unregistering transaction callback for current session"); + UnregisterXactCallback(JITXactCallback, nullptr); + u_sess->mot_cxt.jit_xact_callback_registered = false; + } +} + +extern void ForceJitContextInvalidation(MotJitContext* jitContext) +{ + // Here we don't have necessary locks, because the plan was just invalidated and about to be refreshed (See the + // caller RevalidateCachedQuery). We should not purge here without locks (purge tries to free the keys, etc., but + // another DDL can happen in parallel). All the purge should have happened already during the DDL operation. + InvalidateJitContext(jitContext, 0, JIT_CONTEXT_INVALID); +} + +extern bool IsInvokeQueryPlan(CachedPlanSource* planSource, Oid* functionOid, TransactionId* functionTxnId) +{ + if (planSource->query_list == nullptr) { + return false; + } + + if (list_length(planSource->query_list) != 1) { + return false; + } + + Query* query = (Query*)linitial(planSource->query_list); + FuncExpr* funcExpr = GetFuncExpr(query); + if (funcExpr == nullptr) { + return false; + } + + Oid funcId = funcExpr->funcid; + TransactionId funcXmin = InvalidTransactionId; + HeapTuple procTuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcId)); + if (HeapTupleIsValid(procTuple)) { + funcXmin = HeapTupleGetRawXmin(procTuple); + } + ReleaseSysCache(procTuple); + + if (functionOid != nullptr) { + *functionOid = funcId; + } + if (functionTxnId != nullptr) { + *functionTxnId = funcXmin; + } + return true; +} } // namespace JitExec diff --git a/src/gausskernel/storage/mot/jit_exec/jit_explain.cpp b/src/gausskernel/storage/mot/jit_exec/jit_explain.cpp index 16e488eba..d6b367dd4 100644 --- a/src/gausskernel/storage/mot/jit_exec/jit_explain.cpp +++ b/src/gausskernel/storage/mot/jit_exec/jit_explain.cpp @@ -27,6 +27,9 @@ #include "jit_common.h" #include "mot_engine.h" #include "utilities.h" +#include "utils/timestamp.h" +#include "utils/lsyscache.h" +#include "mot_internal.h" namespace JitExec { DECLARE_LOGGER(JitExplain, JitExec) @@ -38,75 +41,36 @@ static void ExplainRangeSelectPlan(Query* query, JitRangeSelectPlan* plan, bool static void ExplainConstExpr(JitConstExpr* expr) { - MOT_ASSERT(expr->_source_expr->type == T_Const); - Const* constExpr = (Const*)expr->_source_expr; - switch (constExpr->consttype) { - case BOOLOID: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[bool] %u", (unsigned)DatumGetBool(constExpr->constvalue)); - break; - - case CHAROID: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[char] %u", (unsigned)DatumGetChar(constExpr->constvalue)); - break; - - case INT1OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[int1] %u", (unsigned)DatumGetUInt8(constExpr->constvalue)); - break; - - case INT2OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[int2] %u", (unsigned)DatumGetUInt16(constExpr->constvalue)); - break; - - case INT4OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[int4] %u", (unsigned)DatumGetUInt32(constExpr->constvalue)); - break; - - case INT8OID: - MOT_LOG_APPEND( - MOT::LogLevel::LL_TRACE, "[int8] %u" PRIu64, (uint64_t)DatumGetUInt64(constExpr->constvalue)); - break; - - case TIMESTAMPOID: - MOT_LOG_APPEND( - MOT::LogLevel::LL_TRACE, "[timestamp] %" PRIu64, (uint64_t)DatumGetTimestamp(constExpr->constvalue)); - break; - - case FLOAT4OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[float4] %f", (double)DatumGetFloat4(constExpr->constvalue)); - break; - - case FLOAT8OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[float8] %f", (double)DatumGetFloat8(constExpr->constvalue)); - break; - - case VARCHAROID: { - VarChar* vc = DatumGetVarCharPP(constExpr->constvalue); - int size = VARSIZE_ANY_EXHDR(vc); - char* src = VARDATA_ANY(vc); - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[varchar] %.*s", size, src); - } break; - - case NUMERICOID: { - Datum result = DirectFunctionCall1(numeric_out, constExpr->constvalue); - char* cstring = DatumGetCString(result); - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[numeric] %s", cstring); - } break; - - default: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, - "[type %d] %" PRIu64, - (int)constExpr->consttype, - (uint64_t)constExpr->constvalue); - break; + if (expr->_is_null) { // special case of default parameters + PrintDatum(MOT::LogLevel::LL_TRACE, expr->_const_type, PointerGetDatum(nullptr), true); + } else { + MOT_ASSERT(expr->_source_expr != nullptr); + MOT_ASSERT(expr->_source_expr->type == T_Const); + Const* constExpr = (Const*)expr->_source_expr; + PrintDatum(MOT::LogLevel::LL_TRACE, constExpr->consttype, constExpr->constvalue, constExpr->constisnull); } } - +/* function name:ExplainRealColumnName + function purpose:Retrieves the actual column name for a given query based on a table reference ID + and a column ID. This is helpful to decipher real column names especially in complex queries where + joins or aliasing might be involved. + input:query The executed query. tableRefId The table reference ID from the query.columnId The column ID within the table. + output:none + note:none + annotator:liushifa + annotate time:2023/10/05 22:27:42 + contact:3325287047@qq.com +*/ static void ExplainRealColumnName(const Query* query, int tableRefId, int columnId) { + // Acquire the current transaction manager instance + MOT::TxnManager* currTxn = GetSafeTxn(__FUNCTION__); + // Check if the tableRefId is valid + MOT_ASSERT(currTxn != nullptr); if (tableRefId <= list_length(query->rtable)) { // varno index is 1-based RangeTblEntry* rte = (RangeTblEntry*)list_nth(query->rtable, tableRefId - 1); if (rte->rtekind == RTE_RELATION) { - MOT::Table* table = MOT::GetTableManager()->GetTableByExternal(rte->relid); + MOT::Table* table = currTxn->GetTableByExternalId(rte->relid); if (table != nullptr) { if (list_length(query->rtable) == 1) { MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "%s", table->GetFieldName(columnId)); @@ -121,7 +85,7 @@ static void ExplainRealColumnName(const Query* query, int tableRefId, int column tableRefId = aliasVar->varno; if (tableRefId <= list_length(query->rtable)) { // tableRefId is 1-based rte = (RangeTblEntry*)list_nth(query->rtable, tableRefId - 1); - MOT::Table* table = MOT::GetTableManager()->GetTableByExternal(rte->relid); + MOT::Table* table = currTxn->GetTableByExternalId(rte->relid); if (table != nullptr) { // take real column id and not column id from virtual join table columnId = aliasVar->varattno; @@ -177,68 +141,36 @@ static void ExplainParamExpr(JitParamExpr* expr) } } -#define APPLY_UNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, #name "("); \ - ExplainExpr(query, plan, expr->_args[0]); \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ")"); \ - break; +static void ExplainPGFunction(Query* query, JitPlan* plan, Oid functionId, JitExpr** args, int argCount) +{ + // get PG function name + char* functionName = get_func_name(functionId); + if (functionName != nullptr) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "<%s %u>(", functionName, (unsigned)functionId); + } else { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "(", (unsigned)functionId); + } + pfree(functionName); -#define APPLY_BINARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, #name "("); \ - ExplainExpr(query, plan, expr->_args[0]); \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); \ - ExplainExpr(query, plan, expr->_args[1]); \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ")"); \ - break; - -#define APPLY_TERNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, #name "("); \ - ExplainExpr(query, plan, expr->_args[0]); \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); \ - ExplainExpr(query, plan, expr->_args[1]); \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); \ - ExplainExpr(query, plan, expr->_args[2]); \ - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ")"); \ - break; - -#define APPLY_UNARY_CAST_OPERATOR(funcid, name) APPLY_UNARY_OPERATOR(funcid, name) -#define APPLY_BINARY_CAST_OPERATOR(funcid, name) APPLY_BINARY_OPERATOR(funcid, name) -#define APPLY_TERNARY_CAST_OPERATOR(funcid, name) APPLY_TERNARY_OPERATOR(funcid, name) + for (int i = 0; i < argCount; ++i) { + ExplainExpr(query, plan, args[i]); + if ((i + 1) < argCount) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); + } + } + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ")"); +} static void ExplainOpExpr(Query* query, JitPlan* plan, JitOpExpr* expr) { - // explain the operator - switch (expr->_op_func_id) { - APPLY_OPERATORS() - - default: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[op_%u]", expr->_op_func_id); - break; - } + ExplainPGFunction(query, plan, expr->_op_func_id, expr->_args, expr->_arg_count); } static void ExplainFuncExpr(Query* query, JitPlan* plan, JitFuncExpr* expr) { - // explain the function - switch (expr->_func_id) { - APPLY_OPERATORS() - - default: - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[func_%d]", expr->_func_id); - break; - } + ExplainPGFunction(query, plan, expr->_func_id, expr->_args, expr->_arg_count); } -#undef APPLY_UNARY_OPERATOR -#undef APPLY_BINARY_OPERATOR -#undef APPLY_TERNARY_OPERATOR -#undef APPLY_UNARY_CAST_OPERATOR -#undef APPLY_BINARY_CAST_OPERATOR -#undef APPLY_TERNARY_CAST_OPERATOR - static void ExplainSubLinkExpr(Query* query, JitPlan* plan, JitSubLinkExpr* subLinkExpr) { // currently we support only one sub-query (so we don't use sub-query plan index) @@ -261,6 +193,25 @@ static void ExplainSubLinkExpr(Query* query, JitPlan* plan, JitSubLinkExpr* subL } } +static void ExplainScalarArrayOpExpr(Query* query, JitPlan* plan, JitScalarArrayOpExpr* expr) +{ + ExplainExpr(query, plan, expr->m_scalar); + if (expr->m_useOr) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " IN ANY "); + } else { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " IN ALL "); + } + + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "["); + for (int i = 0; i < expr->m_arraySize; ++i) { + ExplainExpr(query, plan, expr->m_arrayElements[i]); + if ((i + 1) < expr->m_arraySize) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); + } + } + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "]"); +} + static void ExplainExpr(Query* query, JitPlan* plan, JitExpr* expr) { switch (expr->_expr_type) { @@ -288,6 +239,10 @@ static void ExplainExpr(Query* query, JitPlan* plan, JitExpr* expr) ExplainSubLinkExpr(query, plan, (JitSubLinkExpr*)expr); break; + case JIT_EXPR_TYPE_SCALAR_ARRAY_OP: + ExplainScalarArrayOpExpr(query, plan, (JitScalarArrayOpExpr*)expr); + break; + default: MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "[expr]"); break; @@ -312,19 +267,24 @@ static const char* JitWhereOperatorClassToString(JitWhereOperatorClass opClass) } } -static void ExplainFilterArray( - Query* query, JitPlan* plan, int indent, JitFilterArray* filterArray, bool isSubQuery = false) +static void ExplainFilterArray(Query* query, JitPlan* plan, int indent, JitFilterArray* filterArray, + bool isSubQuery = false, bool isOneTime = false) { if (!isSubQuery) { - MOT_LOG_BEGIN(MOT::LogLevel::LL_TRACE, "%*sFILTER ON (", indent, ""); + if (isOneTime) { + MOT_LOG_BEGIN(MOT::LogLevel::LL_TRACE, "%*sONE-TIME FILTER ON (", indent, ""); + } else { + MOT_LOG_BEGIN(MOT::LogLevel::LL_TRACE, "%*sFILTER ON (", indent, ""); + } } else { - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " FILTER ON ("); + if (isOneTime) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " FILTER ON ("); + } else { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " ONE-TIME FILTER ON ("); + } } for (int i = 0; i < filterArray->_filter_count; ++i) { - ExplainExpr(query, plan, filterArray->_scan_filters[i]._lhs_operand); - JitWhereOperatorClass opClass = ClassifyWhereOperator(filterArray->_scan_filters[i]._filter_op); - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " %s ", JitWhereOperatorClassToString(opClass)); - ExplainExpr(query, plan, filterArray->_scan_filters[i]._rhs_operand); + ExplainExpr(query, plan, filterArray->_scan_filters[i]); if (i < (filterArray->_filter_count - 1)) { MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " AND "); } @@ -372,7 +332,7 @@ static void ExplainSelectExprArray(Query* query, JitPlan* plan, JitSelectExprArr MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "("); for (int i = 0; i < exprArray->_count; ++i) { MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "%%%d = ", exprArray->_exprs[i]._tuple_column_id); - ExplainExpr(query, plan, (JitExpr*)exprArray->_exprs[i]._column_expr); + ExplainExpr(query, plan, (JitExpr*)exprArray->_exprs[i]._expr); if (i < (exprArray->_count - 1)) { MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); } @@ -435,6 +395,10 @@ static void ExplainPointQueryPlan(Query* query, JitPointQueryPlan* plan, bool is MOT_LOG_TRACE("[Plan] Point Query"); } int indent = 0; + if (plan->_query.m_oneTimeFilters._filter_count > 0) { + ExplainFilterArray(query, (JitPlan*)plan, indent, &plan->_query.m_oneTimeFilters, isSubQuery, true); + indent += 2; + } if (plan->_query._filters._filter_count > 0) { ExplainFilterArray(query, (JitPlan*)plan, indent, &plan->_query._filters, isSubQuery); indent += 2; @@ -556,11 +520,15 @@ static const char* JitIndexScanDirectionToString(JitIndexScanDirection scanDirec static void ExplainIndexScan(Query* query, JitPlan* plan, int indent, JitIndexScan* indexScan, const char* scanName = "", bool isSubQuery = false) { + if (indexScan->m_oneTimeFilters._filter_count > 0) { + ExplainFilterArray(query, plan, indent, &indexScan->m_oneTimeFilters, isSubQuery, true); + indent += 2; + } if (indexScan->_filters._filter_count > 0) { ExplainFilterArray(query, plan, indent, &indexScan->_filters, isSubQuery); indent += 2; } - MOT::Index* index = indexScan->_table->GetIndex(indexScan->_index_id); + MOT::Index* index = indexScan->_index; if (isSubQuery) { MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "%s%sSCAN %s index %s (%s, %s) ON (", @@ -633,10 +601,15 @@ static void ExplainAggregateOperator(int indent, const JitAggregate* aggregate, if (aggregate->_distinct) { MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "DISTINCT("); } - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, - "%s.%s)", - aggregate->_table->GetTableName().c_str(), - aggregate->_table->GetFieldName(aggregate->_table_column_id)); + if (aggregate->_table == nullptr) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "*)"); + } else { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, + "%s.%s type %d)", + aggregate->_table->GetTableName().c_str(), + aggregate->_table->GetFieldName(aggregate->_table_column_id), + aggregate->_element_type); + } if (aggregate->_distinct) { MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ")"); } @@ -645,6 +618,46 @@ static void ExplainAggregateOperator(int indent, const JitAggregate* aggregate, } } +static void ExplainColumnName(Query* query, JitExpr* expr) +{ + if (expr->_expr_type == JIT_EXPR_TYPE_VAR) { + Var* varExpr = (Var*)expr->_source_expr; + int columnId = varExpr->varattno; + int tableRefId = varExpr->varno; + ExplainRealColumnName(query, tableRefId, columnId); + } +} + +static void ExplainRangeSelectSort(Query* query, JitRangeSelectPlan* plan, int indent) +{ + JitNonNativeSortParams* sortParams = plan->m_nonNativeSortParams; + MOT_LOG_BEGIN(MOT::LogLevel::LL_TRACE, "%*sSORT ", indent, ""); + if (ScanDirectionIsBackward(sortParams->scanDir)) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "DESC"); + } else if (ScanDirectionIsForward(sortParams->scanDir)) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "ASC"); + } + for (int i = 0; i < sortParams->numCols; ++i) { + AttrNumber colId = sortParams->sortColIdx[i]; + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " {Column %d [", colId); + ExplainColumnName(query, plan->_select_exprs._exprs[colId - 1]._expr); + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, + "], SortOp %u, Collation %u, nulls-first %s}", + sortParams->sortOperators[i], + sortParams->collations[i], + sortParams->nullsFirst[i] ? "true" : "false"); + if (i + 1 < plan->m_nonNativeSortParams->numCols) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ","); + } + } + if (plan->m_nonNativeSortParams->bound == 0) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " no bound"); + } else { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " bound %" PRIu64, plan->m_nonNativeSortParams->bound); + } + MOT_LOG_END(MOT::LogLevel::LL_TRACE); +} + static void ExplainRangeSelectPlan(Query* query, JitRangeSelectPlan* plan, bool isSubQuery /* = false */) { if (isSubQuery) { @@ -654,9 +667,13 @@ static void ExplainRangeSelectPlan(Query* query, JitRangeSelectPlan* plan, bool MOT_LOG_TRACE("[Plan] Range SELECT from table %s:", plan->_index_scan._table->GetTableName().c_str()); } int indent = 0; - if (plan->_aggregate._aggreaget_op != JIT_AGGREGATE_NONE) { + for (int aggIndex = 0; aggIndex < plan->m_aggCount; ++aggIndex) { indent += 2; - ExplainAggregateOperator(indent, &plan->_aggregate, isSubQuery); + ExplainAggregateOperator(indent, &plan->m_aggregates[aggIndex], isSubQuery); + if (aggIndex + 1 < plan->m_aggCount) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); + } + indent -= 2; } if (plan->_limit_count > 0) { if (isSubQuery) { @@ -666,9 +683,13 @@ static void ExplainRangeSelectPlan(Query* query, JitRangeSelectPlan* plan, bool MOT_LOG_TRACE("%*sLIMIT %d", indent, "", plan->_limit_count); } } - if (plan->_aggregate._aggreaget_op == JIT_AGGREGATE_NONE) { + if (plan->m_nonNativeSortParams != nullptr) { + indent += 2; + ExplainRangeSelectSort(query, plan, indent); + } + if (plan->m_aggCount == 0) { if (isSubQuery) { - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " SELECT"); + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "%*sSELECT", indent, ""); ExplainSelectExprArray(query, (JitPlan*)plan, &plan->_select_exprs); } else { indent += 2; @@ -681,18 +702,43 @@ static void ExplainRangeSelectPlan(Query* query, JitRangeSelectPlan* plan, bool ExplainIndexScan(query, (JitPlan*)plan, indent + 2, &plan->_index_scan, "", isSubQuery); } +static void ExplainRangeDeletePlan(Query* query, JitRangeDeletePlan* plan) +{ + MOT_LOG_TRACE("[Plan] Range DELETE table %s:", plan->_index_scan._table->GetTableName().c_str()); + ExplainIndexScan(query, (JitPlan*)plan, 2, &plan->_index_scan); +} + static void ExplainRangeScanPlan(Query* query, JitRangeScanPlan* plan) { if (plan->_command_type == JIT_COMMAND_UPDATE) { ExplainRangeUpdatePlan(query, (JitRangeUpdatePlan*)plan); } else if (plan->_command_type == JIT_COMMAND_SELECT) { ExplainRangeSelectPlan(query, (JitRangeSelectPlan*)plan); + } else if (plan->_command_type == JIT_COMMAND_DELETE) { + ExplainRangeDeletePlan(query, (JitRangeDeletePlan*)plan); } else { MOT_LOG_TRACE("Invalid plan command type"); return; } } +static const char* JitJoinTypeToString(JitJoinType joinType) +{ + switch (joinType) { + case JitJoinType::JIT_JOIN_INNER: + return "INNER"; + case JitJoinType::JIT_JOIN_LEFT: + return "LEFT"; + case JitJoinType::JIT_JOIN_FULL: + return "FULL"; + case JitJoinType::JIT_JOIN_RIGHT: + return "RIGHT"; + case JitJoinType::JIT_JOIN_INVALID: + default: + return "Invalid"; + } +} + static const char* JitJoinScanTypeToString(JitJoinScanType scanType) { switch (scanType) { @@ -712,15 +758,21 @@ static const char* JitJoinScanTypeToString(JitJoinScanType scanType) static void ExplainJoinPlan(Query* query, JitJoinPlan* plan) { - MOT_LOG_TRACE("[Plan] JOIN table %s on table %s (%s)", + MOT_LOG_TRACE("[Plan] %s JOIN table %s on table %s (%s)", + JitJoinTypeToString(plan->_join_type), plan->_outer_scan._table->GetTableName().c_str(), plan->_inner_scan._table->GetTableName().c_str(), JitJoinScanTypeToString(plan->_scan_type)); int indent = 0; - if (plan->_aggregate._aggreaget_op != JIT_AGGREGATE_NONE) { + for (int aggIndex = 0; aggIndex < plan->m_aggCount; ++aggIndex) { indent += 2; - ExplainAggregateOperator(indent, &plan->_aggregate); - } else { + ExplainAggregateOperator(indent, &plan->m_aggregates[aggIndex]); + if (aggIndex + 1 < plan->m_aggCount) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); + } + indent -= 2; + } + if (plan->m_aggCount == 0) { indent += 2; MOT_LOG_BEGIN(MOT::LogLevel::LL_TRACE, "%*sSELECT", indent, ""); ExplainSelectExprArray(query, (JitPlan*)plan, &plan->_select_exprs); @@ -749,6 +801,34 @@ static void ExplainCompoundPlan(Query* query, JitCompoundPlan* plan) ExplainSearchExprArray(query, (JitPlan*)plan, indent + 4, &plan->_outer_query_plan->_query._search_exprs, false); } +static void ExplainInvokePlan(Query* query, JitInvokePlan* plan) +{ + MOT_LOG_TRACE("[Plan] Invoke stored procedure:"); + MOT_LOG_BEGIN(MOT::LogLevel::LL_TRACE, " %s[id=%u] (", plan->_function_name, plan->_function_id); + for (int i = 0; i < plan->_arg_count; ++i) { + ExplainExpr(query, (JitPlan*)plan, plan->_args[i]); + if (i < plan->_arg_count - 1) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); + } + } + if ((plan->_arg_count > 0) && (plan->m_defaultParamCount > 0)) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); + } + for (int i = 0; i < plan->m_defaultParamCount; ++i) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, " "); + if (plan->m_defaultParams[i] == nullptr) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "null"); + } else { + ExplainExpr(query, (JitPlan*)plan, plan->m_defaultParams[i]); + } + if (i < plan->m_defaultParamCount - 1) { + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ", "); + } + } + MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, ")"); + MOT_LOG_END(MOT::LogLevel::LL_TRACE); +} + extern void JitExplainPlan(Query* query, JitPlan* plan) { if (plan != nullptr) { @@ -773,10 +853,19 @@ extern void JitExplainPlan(Query* query, JitPlan* plan) ExplainCompoundPlan(query, (JitCompoundPlan*)plan); break; + case JIT_PLAN_INVOKE: + ExplainInvokePlan(query, (JitInvokePlan*)plan); + break; + case JIT_PLAN_INVALID: default: break; } } } + +extern void JitExplainPlan(PLpgSQL_function* function, JitPlan* plan) +{ + // not implemented yet +} } // namespace JitExec diff --git a/src/gausskernel/storage/mot/jit_exec/jit_helpers.cpp b/src/gausskernel/storage/mot/jit_exec/jit_helpers.cpp index 2aa3fd7a3..95d11514d 100644 --- a/src/gausskernel/storage/mot/jit_exec/jit_helpers.cpp +++ b/src/gausskernel/storage/mot/jit_exec/jit_helpers.cpp @@ -32,6 +32,14 @@ #include "access/xact.h" #include "utils/array.h" #include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/lsyscache.h" +#include "catalog/pg_type.h" +#include "fmgr.h" +#include "parser/parse_coerce.h" +#include "access/tupconvert.h" +#include "opfusion/opfusion.h" +#include "access/tableam.h" #include "jit_helpers.h" #include "jit_common.h" @@ -39,35 +47,120 @@ #include "utilities.h" #include -typedef std::unordered_set DistinctIntSetType; -typedef std::unordered_set DistinctDoubleSetType; +#include "storage/mot/mot_fdw.h" +#include "jit_llvm_util.h" +#include "jit_plan_sp.h" +#include "jit_profiler.h" +#include "jit_source.h" +#include "mot_fdw_helpers.h" DECLARE_LOGGER(LlvmHelpers, JitExec) +struct NumericEquals : public std::binary_function { + inline bool operator()(const Datum& lhs, const Datum& rhs) const + { + Datum res = DirectFunctionCall2(numeric_eq, lhs, rhs); + return DatumGetBool(res); + } +}; + +struct NumericHash : public std::unary_function { + inline size_t operator()(const Datum& arg) const + { + Numeric num = DatumGetNumeric(arg); + NumericDigit* digits = NUMERIC_DIGITS(num); + int ndigits = NUMERIC_NDIGITS(num); + int nweight = NUMERIC_WEIGHT(num); + int dscale = NUMERIC_DSCALE(num); + int nsign = NUMERIC_SIGN(num); + + size_t hashValue = 17; + hashValue = hashValue * 19 + ndigits; + hashValue = hashValue * 19 + nweight; + hashValue = hashValue * 19 + dscale; + hashValue = hashValue * 19 + nsign; + for (int i = 0; i < ndigits; ++i) { + hashValue = hashValue * 19 + digits[i]; + } + return hashValue; + } +}; + +struct VarcharEquals : public std::binary_function { + inline bool operator()(const Datum& lhs, const Datum& rhs) const + { + Datum res = DirectFunctionCall2(bpchareq, lhs, rhs); + return DatumGetBool(res); + } +}; + +struct VarcharHash : public std::unary_function { + inline size_t operator()(const Datum& arg) const + { + bytea* text = DatumGetByteaP(arg); + uint32_t size = VARSIZE(text); // includes header len VARHDRSZ + char* src = VARDATA(text); + uint32_t strSize = size - VARHDRSZ; + + // implement djb2 + size_t hashValue = 5381; + for (uint32_t i = 0; i < strSize; ++i) { + size_t c = (size_t)src[i]; + hashValue = ((hashValue << 5) + hashValue) + c; // hash * 33 + c + } + return hashValue; + } +}; + +typedef std::unordered_set DistinctIntSetType; +typedef std::unordered_set DistinctDoubleSetType; +using DistinctNumericSetType = std::unordered_set; +using DistinctVarcharSetType = std::unordered_set; + /** @brief Helper to prepare a numeric zero. */ static Datum makeNumericZero() { return DirectFunctionCall1(int4_numeric, Int32GetDatum(0)); } -/** @brief Convert numeric Datum to double precision value. */ -static double numericToDouble(Datum numeric_value) -{ - return (double)DatumGetFloat8(DirectFunctionCall1(numeric_float8, numeric_value)); -} - /** @brief Allocates and initializes a distinct set of integers. */ static void* prepareDistinctIntSet() { void* buf = MOT::MemSessionAlloc(sizeof(DistinctIntSetType)); - return new (buf) DistinctIntSetType(); + if (buf) { + buf = new (buf) DistinctIntSetType(); + } + return buf; } /** @brief Allocates and initializes a distinct set of double-precision values. */ static void* prepareDistinctDoubleSet() { void* buf = MOT::MemSessionAlloc(sizeof(DistinctDoubleSetType)); - return new (buf) DistinctDoubleSetType(); + if (buf) { + buf = new (buf) DistinctDoubleSetType(); + } + return buf; +} + +/** @brief Allocates and initializes a distinct set of numeric datum values. */ +static void* prepareDistinctNumericSet() +{ + void* buf = MOT::MemSessionAlloc(sizeof(DistinctNumericSetType)); + if (buf) { + buf = new (buf) DistinctNumericSetType(); + } + return buf; +} + +/** @brief Allocates and initializes a distinct set of varchar datum values. */ +static void* prepareDistinctVarcharSet() +{ + void* buf = MOT::MemSessionAlloc(sizeof(DistinctVarcharSetType)); + if (buf) { + buf = new (buf) DistinctVarcharSetType(); + } + return buf; } /** @brief Inserts an integer to a distinct set of integers. */ @@ -81,7 +174,7 @@ static int insertDistinctIntItem(void* distinct_set, int64_t item) return result; } -/** @brief Inserts an integer to a distinct set of double-precision values. */ +/** @brief Inserts a double-precision number to a distinct set of double-precision values. */ static int insertDistinctDoubleItem(void* distinct_set, double item) { int result = 0; @@ -92,230 +185,67 @@ static int insertDistinctDoubleItem(void* distinct_set, double item) return result; } +/** @brief Inserts a numeric to a distinct set of double-precision values. */ +static int insertDistinctNumericItem(void* distinctSet, Datum item) +{ + int result = 0; + DistinctNumericSetType* numericSet = (DistinctNumericSetType*)distinctSet; + if (numericSet->insert(item).second) { + result = 1; + } + return result; +} + +/** @brief Inserts a varchar to a distinct set of double-precision values. */ +static int insertDistinctVarcharItem(void* distinctSet, Datum item) +{ + int result = 0; + DistinctVarcharSetType* varcharSet = (DistinctVarcharSetType*)distinctSet; + if (varcharSet->insert(item).second) { + MOT_LOG_DEBUG("Varchar Item inserted"); + result = 1; + } + return result; +} + /** @brief Destroys and frees a distinct set of integers. */ static void destroyDistinctIntSet(void* distinct_set) { DistinctIntSetType* int_set = (DistinctIntSetType*)distinct_set; - int_set->~DistinctIntSetType(); - MOT::MemSessionFree(distinct_set); + if (int_set) { + int_set->~DistinctIntSetType(); + MOT::MemSessionFree(distinct_set); + } } /** @brief Destroys and frees a distinct set of double-precision values. */ static void destroyDistinctDoubleSet(void* distinct_set) { DistinctDoubleSetType* double_set = (DistinctDoubleSetType*)distinct_set; - double_set->~DistinctDoubleSetType(); - MOT::MemSessionFree(distinct_set); -} - -/*--------------------------- DEBUG Print Helpers ---------------------------*/ -#ifdef MOT_JIT_DEBUG -/** @brief Prints to log a numeric value. */ -static void dbg_print_numeric(const char* msg, Numeric num) -{ - NumericDigit* digits = NUMERIC_DIGITS(num); - int ndigits; - int i; - - ndigits = NUMERIC_NDIGITS(num); - - MOT_LOG_BEGIN(MOT::LogLevel::LL_DEBUG, "%s: NUMERIC w=%d d=%d ", msg, NUMERIC_WEIGHT(num), NUMERIC_DSCALE(num)); - switch (NUMERIC_SIGN(num)) { - case NUMERIC_POS: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "POS"); - break; - - case NUMERIC_NEG: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "NEG"); - break; - - case NUMERIC_NAN: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "NaN"); - break; - - default: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "SIGN=0x%x", NUMERIC_SIGN(num)); - break; - } - - for (i = 0; i < ndigits; i++) { - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, " %0*d", DEC_DIGITS, digits[i]); - } - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, " (%f)", numericToDouble(NumericGetDatum(num))); - MOT_LOG_END(MOT::LogLevel::LL_DEBUG); -} - -/** @brief Prints to log a varchar value. */ -static void dbg_print_varchar(const char* msg, VarChar* vc) -{ - size_t size = VARSIZE(vc); - char* src = VARDATA(vc); - MOT_LOG_DEBUG("%s: size=%u, data=%.*s", msg, (unsigned)size, (int)size, src); - size = VARSIZE_ANY_EXHDR(vc); - src = VARDATA_ANY(vc); - MOT_LOG_DEBUG("%s: [PG] size=%u, data=%.*s", msg, (unsigned)size, (int)size, src); - // NOTE: last printout looks better, make sure this is what gets into the row -} - -/** @var Prints to log a geneirc datum value. */ -static void dbg_print_datum(const char* msg, Oid ptype, Datum datum, bool isnull) -{ - if (isnull) { - MOT_LOG_DEBUG("[type %u] NULL", ptype); - } else if (ptype == NUMERICOID) { - dbg_print_numeric(msg, DatumGetNumeric(datum)); - } else if (ptype == VARCHAROID) { - dbg_print_varchar(msg, DatumGetVarCharPP(datum)); - } else { - MOT_LOG_BEGIN(MOT::LogLevel::LL_DEBUG, "%s: ", msg); - switch (ptype) { - case BOOLOID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[bool] %u", (unsigned)DatumGetBool(datum)); - break; - - case CHAROID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[char] %u", (unsigned)DatumGetChar(datum)); - break; - - case INT1OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[int1] %u", (unsigned)DatumGetUInt8(datum)); - break; - - case INT2OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[int2] %u", (unsigned)DatumGetUInt16(datum)); - break; - - case INT4OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[int4] %u", (unsigned)DatumGetUInt32(datum)); - break; - - case INT8OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[int8] %" PRIu64, (uint64_t)DatumGetUInt64(datum)); - break; - - case TIMESTAMPOID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[timestamp] %" PRIu64, (uint64_t)DatumGetTimestamp(datum)); - break; - - case FLOAT4OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[float4] %f", (double)DatumGetFloat4(datum)); - break; - - case FLOAT8OID: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[float8] %f", (double)DatumGetFloat8(datum)); - break; - - default: - MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "[type %u] %" PRIu64, ptype, (uint64_t)datum); - break; - } - - MOT_LOG_END(MOT::LogLevel::LL_DEBUG); + if (double_set) { + double_set->~DistinctDoubleSetType(); + MOT::MemSessionFree(distinct_set); } } -#endif -// helper debug printing macros -#ifdef MOT_JIT_DEBUG -#define DBG_PRINT_NUMERIC(msg, numeric) \ - if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { \ - dbg_print_numeric(msg, numeric); \ - } -#else -#define DBG_PRINT_NUMERIC(msg, numeric) -#endif - -#ifdef MOT_JIT_DEBUG -#define DBG_PRINT_VARCHAR(msg, vc) \ - if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { \ - dbg_print_varchar(msg, vc); \ - } -#else -#define DBG_PRINT_VARCHAR(msg, vc) -#endif - -#ifdef MOT_JIT_DEBUG -#define DBG_PRINT_DATUM(msg, ptype, datum, isnull) \ - if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { \ - dbg_print_datum(msg, ptype, datum, isnull); \ - } -#else -#define DBG_PRINT_DATUM(msg, ptype, datum, isnull) -#endif - -static Oid column_type_to_pg(MOT::MOT_CATALOG_FIELD_TYPES column_type) +/** @brief Destroys and frees a distinct set of double-precision values. */ +static void destroyDistinctNumericSet(void* distinctSet) { - Oid pg_type = -1; - switch (column_type) { - case MOT::MOT_TYPE_DECIMAL: - pg_type = NUMERICOID; - break; - - case MOT::MOT_TYPE_VARCHAR: - pg_type = VARCHAROID; - break; - - case MOT::MOT_TYPE_CHAR: - pg_type = CHAROID; - break; - - case MOT::MOT_TYPE_TINY: - pg_type = INT1OID; - break; - - case MOT::MOT_TYPE_SHORT: - pg_type = INT2OID; - break; - - case MOT::MOT_TYPE_INT: - pg_type = INT4OID; - break; - - case MOT::MOT_TYPE_LONG: - pg_type = INT8OID; - break; - - case MOT::MOT_TYPE_FLOAT: - pg_type = FLOAT4OID; - break; - - case MOT::MOT_TYPE_DOUBLE: - pg_type = FLOAT8OID; - break; - - case MOT::MOT_TYPE_DATE: - pg_type = DATEOID; - break; - - case MOT::MOT_TYPE_TIME: - pg_type = TIMEOID; - break; - - case MOT::MOT_TYPE_TIMESTAMP: - pg_type = TIMESTAMPOID; - break; - - case MOT::MOT_TYPE_TIMESTAMPTZ: - pg_type = TIMESTAMPTZOID; - break; - - case MOT::MOT_TYPE_INTERVAL: - pg_type = INTERVALOID; - break; - - case MOT::MOT_TYPE_TIMETZ: - pg_type = TIMETZOID; - break; - - case MOT::MOT_TYPE_BLOB: - pg_type = BLOBOID; - break; - - default: - break; + DistinctNumericSetType* numericSet = (DistinctNumericSetType*)distinctSet; + if (numericSet) { + numericSet->~DistinctNumericSetType(); + MOT::MemSessionFree(numericSet); } +} - return pg_type; +/** @brief Destroys and frees a distinct set of varchar values. */ +static void destroyDistinctVarcharSet(void* distinctSet) +{ + DistinctVarcharSetType* varcharSet = (DistinctVarcharSetType*)distinctSet; + if (varcharSet) { + varcharSet->~DistinctVarcharSetType(); + MOT::MemSessionFree(varcharSet); + } } /*--------------------------- LLVM Access Helpers ---------------------------*/ @@ -325,10 +255,89 @@ void debugLog(const char* function, const char* msg) MOT_LOG_DEBUG("%s: %s", function, msg); } +void debugLogInt(const char* msg, int arg) +{ + MOT_LOG_DEBUG(msg, arg); +} + +void debugLogString(const char* msg, const char* arg) +{ + MOT_LOG_DEBUG(msg, arg); +} + +void debugLogStringDatum(const char* msg, int64_t arg) +{ + DEBUG_PRINT_DATUM(msg, VARCHAROID, (Datum)arg, false); +} + +void debugLogDatum(const char* msg, Datum value, int isNull, int type) +{ + DEBUG_PRINT_DATUM(msg, type, (Datum)value, isNull); +} + +static void RaiseLlvmFault(int faultCode) +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext); + jitContext->m_execState->m_faultCode = (uint64_t)faultCode; + MOT_LOG_DEBUG("RaiseLlvmFault, jumping to faultBuf at %p on exec state %p: %s", + (int8_t*)jitContext->m_execState->m_faultBuf, + jitContext->m_execState, + jitContext->m_queryString); + siglongjmp(jitContext->m_execState->m_faultBuf, faultCode); +} + +static void RaiseFault(int faultCode) +{ + RaiseLlvmFault(faultCode); +} + +inline void RaiseAccessViolationFault() +{ + RaiseLlvmFault(LLVM_FAULT_ACCESS_VIOLATION); +} + +inline void RaiseResourceLimitFault() +{ + RaiseLlvmFault(LLVM_FAULT_RESOURCE_LIMIT); +} + +inline void ValidatePointer(void* ptr, const char* file, int line, const char* msg) +{ + if (ptr == nullptr) { + MOT_LOG_ERROR("Invalid pointer accessed at %s, line %d: %s", file, line, msg); + RaiseAccessViolationFault(); + } +} + +inline void ValidateArray(int index, int arraySize, const char* file, int line, const char* msg) +{ + if (index >= arraySize) { + MOT_LOG_ERROR("Array index %d out of bounds %d at %s, line %d: %s", index, arraySize, file, line, msg); + RaiseAccessViolationFault(); + } +} + +/** @define Helper macro for validating pointer access. */ +#define VERIFY_PTR(ptr, msg) ValidatePointer(ptr, __FILE__, __LINE__, msg) + +/** @define Helper macro for validating array access. */ +#define VERIFY_ARRAY_ACCESS(index, arraySize, msg) ValidateArray(index, arraySize, __FILE__, __LINE__, msg) + /*--------------------------- Engine Access Helpers ---------------------------*/ +/* function name:isSoftMemoryLimitReached + function purpose:Checks if the soft memory limit has been reached in the MOT engine. + input:none + output:Returns 1 if the soft memory limit has been reached, otherwise 0. + note:none + annotator:liushifa + annotate time:2023/10/05 22:45:07 + contact:3325287047@qq.com +*/ int isSoftMemoryLimitReached() { int result = 0; + // Check if soft memory limit is reached in the MOT engine if (MOT::MOTEngine::GetInstance()->IsSoftMemoryLimitReached()) { MOT_LOG_TRACE("Memory limit reached, aborting transaction"); result = 1; @@ -362,7 +371,11 @@ MOT::Index* getTableIndex(MOT::Table* table, int index_id) void InitKey(MOT::Key* key, MOT::Index* index) { - MOT_LOG_DEBUG("Initializing key %p by source index %s (%p)", key, index->GetName().c_str(), index); + MOT_LOG_DEBUG("Initializing key %p by source index %s %u (%p)", + key, + index->GetName().c_str(), + (unsigned)index->GetExtId(), + index); key->InitKey(index->GetKeyLength()); MOT_LOG_DEBUG("key %p initialized to %u bytes", key, key->GetKeyLength()); } @@ -379,58 +392,96 @@ MOT::Column* getColumnAt(MOT::Table* table, int table_colid) return column; } -void setExprArgIsNull(int arg_pos, int isnull) +void SetExprIsNull(int isnull) { - MOT_LOG_DEBUG("Setting expression argument %d isnull to: %d", arg_pos, isnull); - u_sess->mot_cxt.jit_context->m_argIsNull[arg_pos] = isnull ? 1 : 0; + MOT_LOG_DEBUG("Setting expression isnull to: %d", isnull); + u_sess->mot_cxt.jit_context->m_execState->m_exprIsNull = isnull; } -int getExprArgIsNull(int arg_pos) +int GetExprIsNull() { - int result = u_sess->mot_cxt.jit_context->m_argIsNull[arg_pos]; - MOT_LOG_DEBUG("Retrieved expression argument %d isnull: %d", arg_pos, result); + int result = (int)u_sess->mot_cxt.jit_context->m_execState->m_exprIsNull; + MOT_LOG_DEBUG("Retrieved expression isnull: %d", result); return result; } -Datum GetConstAt(int constId, int argPos) +void SetExprCollation(int collationId) +{ + MOT_LOG_DEBUG("Setting expression collation to: %d", collationId); + u_sess->mot_cxt.jit_context->m_execState->m_exprCollationId = collationId; +} + +int GetExprCollation() +{ + int result = (int)u_sess->mot_cxt.jit_context->m_execState->m_exprCollationId; + MOT_LOG_DEBUG("Retrieved expression collation: %d", result); + return result; +} + +#ifdef MOT_JIT_DEBUG +#define GET_EXPR_IS_NULL() GetExprIsNull() +#define SET_EXPR_IS_NULL(isnull) SetExprIsNull(isnull) +#define GET_EXPR_COLLATION() GetExprCollation() +#define SET_EXPR_COLLATION(collation) SetExprCollation(collation) +#else +#define GET_EXPR_IS_NULL() (int)u_sess->mot_cxt.jit_context->m_execState->m_exprIsNull +#define SET_EXPR_IS_NULL(isnull) u_sess->mot_cxt.jit_context->m_execState->m_exprIsNull = (isnull) +#define GET_EXPR_COLLATION() (int)u_sess->mot_cxt.jit_context->m_execState->m_exprCollationId +#define SET_EXPR_COLLATION(collation) u_sess->mot_cxt.jit_context->m_execState->m_exprCollationId = (collation) +#endif + +Datum GetConstAt(int constId) { MOT_LOG_DEBUG("Retrieving constant datum by id %d", constId); Datum result = PointerGetDatum(nullptr); - JitExec::JitContext* ctx = u_sess->mot_cxt.jit_context; + JitExec::MotJitContext* ctx = u_sess->mot_cxt.jit_context; if (constId < (int)ctx->m_constDatums.m_datumCount) { JitExec::JitDatum* datum = &ctx->m_constDatums.m_datums[constId]; result = datum->m_datum; - setExprArgIsNull(argPos, datum->m_isNull); - DBG_PRINT_DATUM("Retrieved constant datum", datum->m_type, datum->m_datum, datum->m_isNull); + ctx->m_execState->m_exprIsNull = datum->m_isNull; + DEBUG_PRINT_DATUM("Retrieved constant datum", datum->m_type, datum->m_datum, datum->m_isNull); } else { MOT_LOG_ERROR("Invalid constant identifier: %d", constId); + RaiseAccessViolationFault(); } return result; } -Datum getDatumParam(ParamListInfo params, int paramid, int arg_pos) +Datum getDatumParam(ParamListInfo params, int paramid) { MOT_LOG_DEBUG("Retrieving datum param at index %d", paramid); - DBG_PRINT_DATUM( + DEBUG_PRINT_DATUM( "Param value", params->params[paramid].ptype, params->params[paramid].value, params->params[paramid].isnull); - setExprArgIsNull(arg_pos, params->params[paramid].isnull); + SET_EXPR_IS_NULL(params->params[paramid].isnull); return params->params[paramid].value; } -Datum readDatumColumn(MOT::Table* table, MOT::Row* row, int colid, int arg_pos) +Datum readDatumColumn(MOT::Table* table, MOT::Row* row, int colid, int innerRow, int subQueryIndex) { + // special case: row is null (can happen with left join) + if (row == nullptr) { + MOT_LOG_DEBUG("readDatumColumn(): Row is null (left join?)"); + SET_EXPR_IS_NULL(1); + return PointerGetDatum(NULL); // return proper NULL datum + } + MOT::Column* column = table->GetField(colid); - MOT_LOG_DEBUG("Reading Datum value from row %p column %p", row, column); + MOT_LOG_DEBUG("Reading Datum value from row %p column %p [%s] in table %p [%s]", + row, + column, + column->m_name, + table, + table->GetTableName().c_str()); Datum result = PointerGetDatum(NULL); // return proper NULL datum if column value is null FormData_pg_attribute attr; attr.attnum = colid; - attr.atttypid = column_type_to_pg(column->m_type); + attr.atttypid = ConvertMotColumnTypeToOid(column->m_type); bool isnull = false; MOTAdaptor::MOTToDatum(table, &attr, (uint8_t*)row->GetData(), &result, &isnull); - DBG_PRINT_DATUM("Column value", attr.atttypid, result, isnull); - setExprArgIsNull(arg_pos, (int)isnull); + DEBUG_PRINT_DATUM("Column value", attr.atttypid, result, isnull); + SET_EXPR_IS_NULL((int)isnull); return result; } @@ -438,14 +489,14 @@ Datum readDatumColumn(MOT::Table* table, MOT::Row* row, int colid, int arg_pos) void writeDatumColumn(MOT::Row* row, MOT::Column* column, Datum value) { MOT_LOG_DEBUG("Writing to row %p column %p datum value", row, column); - DBG_PRINT_DATUM("Datum value", column_type_to_pg(column->m_type), value, 0); - MOTAdaptor::DatumToMOT(column, value, column_type_to_pg(column->m_type), (uint8_t*)row->GetData()); + DEBUG_PRINT_DATUM("Datum value", ConvertMotColumnTypeToOid(column->m_type), value, 0); + MOTAdaptor::DatumToMOT(column, value, ConvertMotColumnTypeToOid(column->m_type), (uint8_t*)row->GetData()); } void buildDatumKey( MOT::Column* column, MOT::Key* key, Datum value, int index_colid, int offset, int size, int value_type) { - int isnull = getExprArgIsNull(0); + int isnull = GET_EXPR_IS_NULL(); MOT_LOG_DEBUG("buildKey: writing datum value for index column %d at key buf offset %d (col=%p, key=%p, datum=%p, " "value-type=%d, key-size=%u, col-size=%u, size=%d, is-null: %d)", index_colid, @@ -458,7 +509,7 @@ void buildDatumKey( (unsigned)column->m_size, size, isnull); - DBG_PRINT_DATUM("Key Datum", value_type, value, 0); + DEBUG_PRINT_DATUM("Key Datum", value_type, value, 0); if (isnull) { MOT_LOG_DEBUG("Setting null key datum at offset %d (%d bytes)", offset, size); errno_t erc = memset_s(key->GetKeyBuf() + offset, key->GetKeyLength() - offset, 0x00, size); @@ -467,82 +518,529 @@ void buildDatumKey( MOTAdaptor::DatumToMOTKey(column, (Oid)value_type, value, - column->m_envelopeType, + (Oid)column->m_envelopeType, key->GetKeyBuf() + offset, - column->m_size, + size, KEY_OPER::READ_KEY_EXACT, 0x00); } } /*--------------------------- Invoke PG Operators ---------------------------*/ -// cast operators retain first null parameter as null result -// other operator will crash if null is provided -#define APPLY_UNARY_OPERATOR(funcid, name) \ - Datum invoke_##name(Datum arg, int arg_pos) \ - { \ - MOT_LOG_DEBUG("Invoking unary operator: " #name); \ - return DirectFunctionCall1(name, arg); \ - } - -#define APPLY_UNARY_CAST_OPERATOR(funcid, name) \ - Datum invoke_##name(Datum arg, int arg_pos) \ - { \ - MOT_LOG_DEBUG("Invoking unary cast operator: " #name); \ - int isnull = getExprArgIsNull(arg_pos); \ - return isnull ? arg : DirectFunctionCall1(name, arg); \ - } - -#define APPLY_BINARY_OPERATOR(funcid, name) \ - Datum invoke_##name(Datum lhs, Datum rhs, int arg_pos) \ - { \ - MOT_LOG_DEBUG("Invoking binary operator: " #name); \ - return DirectFunctionCall2(name, lhs, rhs); \ - } - -#define APPLY_BINARY_CAST_OPERATOR(funcid, name) \ - Datum invoke_##name(Datum lhs, Datum rhs, int arg_pos) \ - { \ - MOT_LOG_DEBUG("Invoking binary cast operator: " #name); \ - int lhs_isnull = getExprArgIsNull(arg_pos); \ - return lhs_isnull ? lhs : DirectFunctionCall2(name, lhs, rhs); \ - } - -#define APPLY_TERNARY_OPERATOR(funcid, name) \ - Datum invoke_##name(Datum arg1, Datum arg2, Datum arg3, int arg_pos) \ - { \ - MOT_LOG_DEBUG("Invoking ternary operator: " #name); \ - return DirectFunctionCall3(name, arg1, arg2, arg3); \ - } - -#define APPLY_TERNARY_CAST_OPERATOR(funcid, name) \ - Datum invoke_##name(Datum arg1, Datum arg2, Datum arg3, int arg_pos) \ - { \ - MOT_LOG_DEBUG("Invoking ternary operator: " #name); \ - int arg1_isnull = getExprArgIsNull(arg_pos); \ - return arg1_isnull ? arg1 : DirectFunctionCall3(name, arg1, arg2, arg3); \ - } - -APPLY_OPERATORS() - -#undef APPLY_UNARY_OPERATOR -#undef APPLY_BINARY_OPERATOR -#undef APPLY_TERNARY_OPERATOR -#undef APPLY_UNARY_CAST_OPERATOR -#undef APPLY_BINARY_CAST_OPERATOR -#undef APPLY_TERNARY_CAST_OPERATOR - -MOT::Row* searchRow(MOT::Table* table, MOT::Key* key, int access_mode_value) +static Datum MakeDatumString(const char* message) { - MOT_LOG_DEBUG("Searching row at table %p by key %p", table, key); + size_t strSize = strlen(message); + size_t allocSize = VARHDRSZ + strSize + 1; + bytea* copy = (bytea*)palloc(allocSize); + if (copy == nullptr) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "JIT Execute", "Failed to allocate %u bytes for error datum string", (unsigned)allocSize); + return PointerGetDatum(nullptr); + } + + errno_t erc = memcpy_s(VARDATA(copy), strSize, (uint8_t*)message, strSize); + securec_check(erc, "\0", "\0"); + + SET_VARSIZE(copy, allocSize); + VARDATA(copy)[strSize] = 0; + return PointerGetDatum(copy); +} + +static MOT::RC HandlePGError(JitExec::JitExecState* execState, const char* operation, int spiConnectId = -1) +{ + volatile MOT::RC result = MOT::RC_ERROR; + + // first restore SPI state if ordered to do so + if (spiConnectId >= 0) { + SPI_disconnect(spiConnectId + 1); + SPI_restore_connection(); + } + + // now handle error + volatile ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while %s: %s", operation, edata->message); + ereport(WARNING, + (errmodule(MOD_MOT), + errmsg("Caught exception while %s: %s", operation, edata->message), + errdetail("%s", edata->detail))); + + // prepare error data and SQL state in JIT execution state + // note: make sure all memory is allocated in caller's context + volatile MemoryContext origCxt = JitExec::SwitchToSPICallerContext(); + PG_TRY(); + { + execState->m_errorMessage = PointerGetDatum(nullptr); + execState->m_errorDetail = PointerGetDatum(nullptr); + execState->m_errorHint = PointerGetDatum(nullptr); + execState->m_sqlStateString = PointerGetDatum(nullptr); + + execState->m_errorMessage = MakeDatumString(edata->message); + if (edata->detail != 0) { + execState->m_errorDetail = MakeDatumString(edata->detail); + } + if (edata->hint != 0) { + execState->m_errorHint = MakeDatumString(edata->hint); + } + execState->m_sqlState = edata->sqlerrcode; + if (execState->m_sqlState == ERRCODE_IN_FAILED_SQL_TRANSACTION) { + result = MOT::RC_TXN_ABORTED; + } else if (execState->m_sqlState == 0) { + execState->m_sqlState = ERRCODE_PLPGSQL_ERROR; + } + char sqlStateCode[6] = {}; + JitExec::SqlStateToCode(execState->m_sqlState, sqlStateCode); + execState->m_sqlStateString = MakeDatumString(sqlStateCode); + } + PG_CATCH(); + { + MOT_LOG_PANIC("Failed to allocate memory in error handler"); + } + PG_END_TRY(); + (void)MemoryContextSwitchTo(origCxt); + + // cleanup error state + FlushErrorState(); + FreeErrorData((ErrorData*)edata); + return result; +} + +static void SetDefaultErrorData(JitExec::JitExecState* execState) +{ + // prepare error data and SQL state in JIT execution state + // note: make sure all memory is allocated in caller's context + volatile MemoryContext origCxt = JitExec::SwitchToSPICallerContext(); + PG_TRY(); + { + execState->m_errorMessage = PointerGetDatum(nullptr); + execState->m_errorDetail = PointerGetDatum(nullptr); + execState->m_errorHint = PointerGetDatum(nullptr); + execState->m_sqlStateString = PointerGetDatum(nullptr); + + execState->m_errorMessage = MakeDatumString("Unknown error occurred"); + execState->m_sqlState = ERRCODE_PLPGSQL_ERROR; + char sqlStateCode[6] = {}; + JitExec::SqlStateToCode(execState->m_sqlState, sqlStateCode); + execState->m_sqlStateString = MakeDatumString(sqlStateCode); + } + PG_CATCH(); + { + MOT_LOG_PANIC("Failed to allocate memory while setting default error data"); + } + PG_END_TRY(); + (void)MemoryContextSwitchTo(origCxt); +} + +inline JitExec::JitFunctionExecState* GetFunctionExecState() +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext != nullptr); + MOT_ASSERT(jitContext->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + JitExec::JitFunctionExecState* execState = (JitExec::JitFunctionExecState*)jitContext->m_execState; + MOT_ASSERT(execState != nullptr); + return execState; +} + +static void SignalException(JitExec::MotJitContext* jitContext) +{ + if (jitContext->m_execState->m_sqlState == 0) { + SetDefaultErrorData(jitContext->m_execState); + } + + int faultCode = jitContext->m_execState->m_sqlState; + + MOT_LOG_DEBUG("SignalException, with faultBuf at %p on exec state %p: %s", + (int8_t*)jitContext->m_execState->m_faultBuf, + jitContext->m_execState, + jitContext->m_queryString); + + if (jitContext->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + if (execState->m_exceptionStack == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Execute JIT LLVM Stored Procedure", + "Cannot signal exception: exception stack is empty, while executing %s. Aborting execution.", + u_sess->mot_cxt.jit_context->m_queryString); + RaiseFault(LLVM_FAULT_UNHANDLED_EXCEPTION); + } + + // set exception value and raise exception status + execState->m_exceptionStatus = 1; + execState->m_exceptionValue = faultCode; + + // set exception origin + execState->m_exceptionOrigin = JIT_EXCEPTION_EXTERNAL; + } else { + JitExec::JitExecState* execState = (JitExec::JitExecState*)jitContext->m_execState; + // set exception value and raise exception status + execState->m_exceptionStatus = 1; + execState->m_exceptionValue = faultCode; + RaiseFault(LLVM_FAULT_UNHANDLED_EXCEPTION); + } +} + +inline void HandlePGFunctionError(JitExec::MotJitContext* jitContext, int spiConnectId) +{ + (void)HandlePGError(jitContext->m_execState, "executing PG Function", spiConnectId); +} + +Datum JitInvokePGFunction0(PGFunction fptr, int collationId) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile Datum result = PointerGetDatum(nullptr); + volatile bool signalException = false; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile int spiConnectId = SPI_connectid(); + + FunctionCallInfoData fcinfo; + InitFunctionCallInfoData(fcinfo, NULL, 0, (Oid)collationId, NULL, NULL); + PG_TRY(); + { + result = fptr(&fcinfo); + SET_EXPR_IS_NULL(fcinfo.isnull); + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + return result; +} + +inline void PrepareFmgrInfo(FunctionCallInfoData* fcinfo, FmgrInfo* flinfo, FuncExpr* funcExpr, List* funcArgs, + ListCell* cells, Const* args, Oid* argTypes, int argCount) +{ + funcExpr->xpr.type = T_FuncExpr; + funcExpr->funcvariadic = false; + + // prepare for function arguments a list of constants nodes with corresponding argument types + for (int i = 0; i < argCount; ++i) { + fcinfo->argTypes[i] = argTypes[i]; + args[i].xpr.type = T_Const; + args[i].consttype = argTypes[i]; + cells[i].data.ptr_value = &args[i]; + if (i < argCount - 1) { + cells[i].next = &cells[i + 1]; + } else { + cells[i].next = nullptr; + } + } + + funcArgs->type = T_List; + funcArgs->length = argCount; + funcArgs->head = &cells[0]; + funcArgs->tail = &cells[argCount - 1]; + + funcExpr->args = funcArgs; + flinfo->fn_expr = (fmNodePtr)funcExpr; + fcinfo->flinfo = flinfo; +} + +Datum JitInvokePGFunction1(PGFunction fptr, int collationId, int isStrict, Datum arg, int isnull, Oid argType) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile Datum result = PointerGetDatum(nullptr); + volatile bool signalException = false; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile int spiConnectId = SPI_connectid(); + + FunctionCallInfoData fcinfo1; + InitFunctionCallInfoData(fcinfo1, NULL, 1, (Oid)collationId, NULL, NULL); + PG_TRY(); + { + bool shouldCallFunc = true; + fcinfo1.arg[0] = arg; + fcinfo1.argnull[0] = isnull; + + // some functions require types in flinfo.fn_expr, so we fake it as required + FmgrInfo flinfo = {}; + FuncExpr funcExpr = {}; + List funcArgs = {}; + ListCell cells[1] = {}; + Const args[1] = {}; + Oid argTypes[1] = {argType}; + PrepareFmgrInfo(&fcinfo1, &flinfo, &funcExpr, &funcArgs, cells, args, argTypes, 1); + + if (isStrict > 0) { + for (int i = 0; i < fcinfo1.nargs; i++) { + if (fcinfo1.argnull[i]) { + shouldCallFunc = false; + break; + } + } + } + + // call the function + if (shouldCallFunc) { + result = fptr(&fcinfo1); + SET_EXPR_IS_NULL(fcinfo1.isnull); + } else { + SET_EXPR_IS_NULL(true); + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + return result; +} + +Datum JitInvokePGFunction2(PGFunction fptr, int collationId, int isStrict, Datum arg1, int isnull1, Oid argType1, + Datum arg2, int isnull2, Oid argType2) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile Datum result = PointerGetDatum(nullptr); + volatile bool signalException = false; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile int spiConnectId = SPI_connectid(); + + FunctionCallInfoData fcinfo2; + InitFunctionCallInfoData(fcinfo2, NULL, 2, (Oid)collationId, NULL, NULL); + PG_TRY(); + { + bool shouldCallFunc = true; + fcinfo2.arg[0] = arg1; + fcinfo2.argnull[0] = isnull1; + fcinfo2.arg[1] = arg2; + fcinfo2.argnull[1] = isnull2; + + // some functions require types in flinfo.fn_expr, so we fake it as required + FmgrInfo flinfo = {}; + FuncExpr funcExpr = {}; + List funcArgs = {}; + ListCell cells[2] = {}; + Const args[2] = {}; + Oid argTypes[2] = {argType1, argType2}; + PrepareFmgrInfo(&fcinfo2, &flinfo, &funcExpr, &funcArgs, cells, args, argTypes, 2); + + if (isStrict > 0) { + for (int i = 0; i < fcinfo2.nargs; i++) { + if (fcinfo2.argnull[i]) { + shouldCallFunc = false; + break; + } + } + } + + // call the function + if (shouldCallFunc) { + result = fptr(&fcinfo2); + SET_EXPR_IS_NULL(fcinfo2.isnull); + } else { + SET_EXPR_IS_NULL(true); + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + return result; +} + +Datum JitInvokePGFunction3(PGFunction fptr, int collationId, int isStrict, Datum arg1, int isnull1, Oid argType1, + Datum arg2, int isnull2, Oid argType2, Datum arg3, int isnull3, Oid argType3) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile Datum result = PointerGetDatum(nullptr); + volatile bool signalException = false; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile int spiConnectId = SPI_connectid(); + + FunctionCallInfoData fcinfo3; + InitFunctionCallInfoData(fcinfo3, NULL, 3, (Oid)collationId, NULL, NULL); + PG_TRY(); + { + bool shouldCallFunc = true; + fcinfo3.arg[0] = arg1; + fcinfo3.argnull[0] = isnull1; + fcinfo3.arg[1] = arg2; + fcinfo3.argnull[1] = isnull2; + fcinfo3.arg[2] = arg3; + fcinfo3.argnull[2] = isnull3; + + // some functions require types in flinfo.fn_expr, so we fake it as required + FmgrInfo flinfo = {}; + FuncExpr funcExpr = {}; + List funcArgs = {}; + ListCell cells[3] = {}; + Const args[3] = {}; + Oid argTypes[3] = {argType1, argType2, argType3}; + PrepareFmgrInfo(&fcinfo3, &flinfo, &funcExpr, &funcArgs, cells, args, argTypes, 3); + + // call the function + if (isStrict > 0) { + for (int i = 0; i < fcinfo3.nargs; i++) { + if (fcinfo3.argnull[i] == true) { + shouldCallFunc = false; + break; + } + } + } + if (shouldCallFunc) { + result = fptr(&fcinfo3); + SET_EXPR_IS_NULL(fcinfo3.isnull); + } else { + SET_EXPR_IS_NULL(true); + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + return result; +} + +static Datum JitInvokePGFunctionNImpl(PGFunction fptr, int collationId, int isStrict, Datum* args, int* isnull, + Oid* argTypes, int argCount, Oid functionId) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile Datum result = PointerGetDatum(nullptr); + volatile bool signalException = false; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile ListCell* cells = nullptr; + volatile Const* constArgs = nullptr; + volatile int spiConnectId = SPI_connectid(); + + FunctionCallInfoData fcinfo; + InitFunctionCallInfoData(fcinfo, NULL, argCount, (Oid)collationId, NULL, NULL); + PG_TRY(); + { + bool shouldCallFunc = true; + for (int i = 0; i < argCount; ++i) { + fcinfo.arg[i] = args[i]; + fcinfo.argnull[i] = isnull[i]; + if (isStrict && isnull[i]) { + SET_EXPR_IS_NULL(true); + shouldCallFunc = false; + break; + } + } + + if (shouldCallFunc) { + // some functions require types in flinfo.fn_expr, so we fake it as required + FmgrInfo flinfo = {}; + FuncExpr funcExpr = {}; + List funcArgs = {}; + cells = (ListCell*)MOT::MemSessionAlloc(sizeof(ListCell) * argCount); + constArgs = (Const*)MOT::MemSessionAlloc(sizeof(Const) * argCount); + if ((cells == nullptr) || (constArgs == nullptr)) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_OUT_OF_LOGICAL_MEMORY), + errmsg("MOT/JIT execution cannot call PG function"), + errdetail("Out of session memory"))); + } + PrepareFmgrInfo( + &fcinfo, &flinfo, &funcExpr, &funcArgs, (ListCell*)cells, (Const*)constArgs, argTypes, argCount); + flinfo.fn_oid = functionId; // for invoke unjittable using plpgsql_call_handler + + // call the function + result = fptr(&fcinfo); + SET_EXPR_IS_NULL(fcinfo.isnull); + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + + // cleanup + if (cells != nullptr) { + MOT::MemSessionFree((void*)cells); + } + if (args != nullptr) { + MOT::MemSessionFree((void*)constArgs); + } + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + return result; +} + +Datum JitInvokePGFunctionN( + PGFunction fptr, int collationId, int isStrict, Datum* args, int* isnull, Oid* argTypes, int argCount) +{ + return JitInvokePGFunctionNImpl(fptr, collationId, isStrict, args, isnull, argTypes, argCount, 0); +} + +uint8_t* JitMemSessionAlloc(uint32_t size) +{ + return (uint8_t*)MOT::MemSessionAlloc(size); +} + +void JitMemSessionFree(uint8_t* ptr) +{ + MOT::MemSessionFree(ptr); +} + +MOT::Row* searchRow(MOT::Table* table, MOT::Key* key, int access_mode_value, int innerRow, int subQueryIndex) +{ + MOT_LOG_DEBUG("searchRow(): Searching row at table %p by key %p", table, key); MOT::Row* row = NULL; MOT::TxnManager* curr_txn = u_sess->mot_cxt.jit_txn; MOT::AccessType access_mode = (MOT::AccessType)access_mode_value; - row = curr_txn->RowLookupByKey(table, access_mode, key); + MOT::RC rc = MOT::RC_OK; + + row = curr_txn->RowLookupByKey(table, access_mode, key, rc); if (row == nullptr) { - MOT_LOG_DEBUG("Row not found"); + MOT_LOG_DEBUG("searchRow(): Row not found"); + if (rc != MOT::RC_OK) { + u_sess->mot_cxt.jit_context->m_rc = rc; + return nullptr; + } + } else { + MOT_ASSERT(rc == MOT::RC_OK); + if (access_mode == MOT::AccessType::WR) { + rc = curr_txn->UpdateLastRowState(access_mode); + if (rc != MOT::RC_OK) { + u_sess->mot_cxt.jit_context->m_rc = rc; + row = nullptr; + } else { + row = curr_txn->GetLastAccessedDraft(); + } + } } - MOT_LOG_DEBUG("Returning row: %p", row); + MOT_LOG_DEBUG("searchRow(): Returning row: %p", row); return row; } @@ -564,15 +1062,17 @@ void resetBitmapSet(MOT::BitmapSet* bmp) int writeRow(MOT::Row* row, MOT::BitmapSet* bmp) { MOT::RC rc = MOT::RC_ERROR; - MOT_LOG_DEBUG("Writing row %p to DB", row); + MOT_LOG_DEBUG("writeRow(): Writing row %p to DB", row); MOT::TxnManager* curr_txn = u_sess->mot_cxt.jit_txn; - MOT_LOG_DEBUG("Current txn is: %p", curr_txn); + MOT_LOG_DEBUG("writeRow(): Current txn is: %p", curr_txn); rc = curr_txn->UpdateLastRowState(MOT::AccessType::WR); - MOT_LOG_DEBUG("Row write result: %d", (int)rc); + MOT_LOG_DEBUG("writeRow(): Row write result: %d", (int)rc); if (rc == MOT::RC_OK) { - MOT_LOG_DEBUG("Overwriting row %p with bitset %p", row, bmp); + MOT_LOG_DEBUG("writeRow(): Overwriting row %p with bitset %p", row, bmp); rc = curr_txn->OverwriteRow(row, *bmp); - MOT_LOG_DEBUG("Overwrite row result: %d", (int)rc); + MOT_LOG_DEBUG("writeRow(): Overwrite row result: %d", (int)rc); + } else { + MOT_LOG_DEBUG("writeRow(): Failed to update row %p with rc (%s)", row, MOT::RcToString(rc)); } return (int)rc; } @@ -596,12 +1096,24 @@ MOT::Row* createNewRow(MOT::Table* table) int insertRow(MOT::Table* table, MOT::Row* row) { MOT::RC rc = MOT::RC_ERROR; - MOT_LOG_DEBUG("Inserting row %p to DB", row); + MOT_LOG_DEBUG("insertRow(): Inserting row %p to DB", row); MOT::TxnManager* curr_txn = u_sess->mot_cxt.jit_txn; - MOT_LOG_DEBUG("Current txn is: %p", curr_txn); + MOT_LOG_DEBUG("insertRow(): Current txn is: %p", curr_txn); + uint8_t* bits = (uint8_t*)row->GetData(); + for (uint32_t i = 1; i < table->GetFieldCount(); i++) { + MOT::Column* col = table->GetField(i); + if (!col->GetIsDropped() && col->m_isNotNull && !BITMAP_GET(bits, (i - 1))) { + MOT_LOG_DEBUG("insertRow(): Cannot set null to not-nullable column %d (%s)", i, col->m_name); + if (u_sess->mot_cxt.jit_context != NULL) { + u_sess->mot_cxt.jit_context->m_execState->m_nullViolationTable = table; + u_sess->mot_cxt.jit_context->m_execState->m_nullColumnId = i; + } + return (int)MOT::RC_NULL_VIOLATION; + } + } rc = table->InsertRow(row, curr_txn); if (rc != MOT::RC_OK) { - MOT_LOG_DEBUG("Insert row failed with rc: %d", (int)rc); + MOT_LOG_DEBUG("insertRow(): Insert row failed with rc: %d", (int)rc); } return (int)rc; } @@ -609,12 +1121,12 @@ int insertRow(MOT::Table* table, MOT::Row* row) int deleteRow() { MOT::RC rc = MOT::RC_ERROR; - MOT_LOG_DEBUG("Deleting row from DB"); + MOT_LOG_DEBUG("deleteRow(): Deleting row from DB"); MOT::TxnManager* curr_txn = u_sess->mot_cxt.jit_txn; - MOT_LOG_DEBUG("Current txn is: %p", curr_txn); + MOT_LOG_DEBUG("deleteRow(): Current txn is: %p", curr_txn); rc = curr_txn->DeleteLastRow(); if (rc != MOT::RC_OK) { - MOT_LOG_DEBUG("Delete row failed with rc: %d", (int)rc); + MOT_LOG_DEBUG("deleteRow(): Delete row failed with rc: %d", (int)rc); } return (int)rc; } @@ -634,10 +1146,12 @@ int setConstNullBit(MOT::Table* table, MOT::Row* row, int table_colid, int isnul { MOT_LOG_DEBUG("Setting const null bit at table %p row %p column id %d: isnull=%d", table, row, table_colid, isnull); - if (isnull && table->GetField(table_colid)->m_isNotNull) { - MOT_LOG_DEBUG("Cannot set null to not-nullable column %d", table_colid); + MOT::Column* column = table->GetField(table_colid); + if (isnull && column->m_isNotNull) { + MOT_LOG_DEBUG("Cannot set null to not-nullable column %d (%s)", table_colid, column->m_name); if (u_sess->mot_cxt.jit_context != NULL) { - u_sess->mot_cxt.jit_context->m_nullColumnId = table_colid; + u_sess->mot_cxt.jit_context->m_execState->m_nullViolationTable = table; + u_sess->mot_cxt.jit_context->m_execState->m_nullColumnId = table_colid; } return (int)MOT::RC_NULL_VIOLATION; } @@ -655,7 +1169,7 @@ int setConstNullBit(MOT::Table* table, MOT::Row* row, int table_colid, int isnul int setExprResultNullBit(MOT::Table* table, MOT::Row* row, int table_colid) { - int isnull = getExprArgIsNull(0); // final result is always put in arg zero + int isnull = GET_EXPR_IS_NULL(); MOT_LOG_DEBUG("Setting expression result isnull at column %d to: %d", table_colid, isnull); return setConstNullBit(table, row, table_colid, isnull); } @@ -672,6 +1186,29 @@ void execStoreVirtualTuple(TupleTableSlot* slot) ::ExecStoreVirtualTuple(slot); } +struct SelectRowFunctor { + MOT::Table* m_table; + MOT::Row* m_row; + TupleTableSlot* m_slot; + int m_tableColumnId; + int m_tupleColumnId; + + SelectRowFunctor(MOT::Table* table, MOT::Row* row, TupleTableSlot* slot, int tableColumnId, int tupleColumnId) + : m_table(table), m_row(row), m_slot(slot), m_tableColumnId(tableColumnId), m_tupleColumnId(tupleColumnId) + {} + + inline void operator()() + { + uint8_t* rowData = const_cast(m_row->GetData()); + m_slot->tts_tupleDescriptor->attrs[m_tupleColumnId].attnum = m_tableColumnId; + MOTAdaptor::MOTToDatum(m_table, + &m_slot->tts_tupleDescriptor->attrs[m_tupleColumnId], + rowData, + &(m_slot->tts_values[m_tupleColumnId]), + &(m_slot->tts_isnull[m_tupleColumnId])); + } +}; + void selectColumn(MOT::Table* table, MOT::Row* row, TupleTableSlot* slot, int table_colid, int tuple_colid) { MOT_LOG_DEBUG("Selecting into tuple column %d from table %s, row %p column id %d [%s]", @@ -681,21 +1218,21 @@ void selectColumn(MOT::Table* table, MOT::Row* row, TupleTableSlot* slot, int ta table_colid, table->GetFieldName(table_colid)); uint8_t* rowData = const_cast(row->GetData()); - slot->tts_tupleDescriptor->attrs[tuple_colid]->attnum = table_colid; + slot->tts_tupleDescriptor->attrs[tuple_colid].attnum = table_colid; MOTAdaptor::MOTToDatum(table, - slot->tts_tupleDescriptor->attrs[tuple_colid], + &slot->tts_tupleDescriptor->attrs[tuple_colid], rowData, &(slot->tts_values[tuple_colid]), &(slot->tts_isnull[tuple_colid])); - DBG_PRINT_DATUM("Column Datum", - slot->tts_tupleDescriptor->attrs[tuple_colid]->atttypid, + DEBUG_PRINT_DATUM("Column Datum", + slot->tts_tupleDescriptor->attrs[tuple_colid].atttypid, slot->tts_values[tuple_colid], slot->tts_isnull[tuple_colid]); } void setTpProcessed(uint64_t* tp_processed, uint64_t rows) { - MOT_LOG_DEBUG("Setting tp_processed at %p to %" PRIu64 "", tp_processed, rows); + MOT_LOG_DEBUG("Setting tp_processed at %p to %" PRIu64 " rows", tp_processed, rows); *tp_processed = rows; } @@ -727,7 +1264,8 @@ void FillKeyPattern(MOT::Key* key, unsigned char pattern, int offset, int size) offset, size, MOT::HexStr(key->GetKeyBuf(), key->GetKeyLength()).c_str()); - key->FillPattern(pattern, size, offset); + MOT_ASSERT((offset + size) <= key->GetKeyLength()); + (void)key->FillPattern(pattern, size, offset); MOT_LOG_DEBUG("Filled key %p pattern %u at offset %d, size %d: %s", key, (unsigned)pattern, @@ -756,19 +1294,28 @@ MOT::IndexIterator* searchIterator(MOT::Index* index, MOT::Key* key, int forward bool forwardDirection = forward_scan ? true : false; bool found = false; - MOT_LOG_DEBUG("Creating begin iterator for index %p from key %p (include_bound=%s): %s", + MOT_LOG_DEBUG("searchIterator(): Creating begin iterator for index %p from key %p (include_bound=%s): %s", index, key, include_bound ? "true" : "false", MOT::HexStr(key->GetKeyBuf(), key->GetKeyLength()).c_str()); MOT::TxnManager* curr_txn = u_sess->mot_cxt.jit_txn; - MOT_LOG_DEBUG("searchIterator: Current txn is: %p", curr_txn); + MOT_LOG_DEBUG("searchIterator(): Current txn is: %p", curr_txn); itr = index->Search(key, matchKey, forwardDirection, curr_txn->GetThdId(), found); - if (!found) { - MOT_LOG_DEBUG("searchIterator: Exact match not found, still continuing, itr=%p", itr); + if (itr != nullptr) { + if (!found) { + MOT_LOG_DEBUG("searchIterator(): Exact match not found, still continuing, itr=%p", itr); + } else { + MOT_LOG_DEBUG("searchIterator(): Exact match found with iterator %p", itr); + } } else { - MOT_LOG_DEBUG("searchIterator: Exact match found with iterator %p", itr); + MOT::RC rootRc = MOT_GET_ROOT_ERROR_RC(); + u_sess->mot_cxt.jit_context->m_rc = rootRc != MOT::RC_OK ? rootRc : MOT::RC_MEMORY_ALLOCATION_ERROR; + MOT_LOG_ERROR("searchIterator(): Failed to create iterator for index (%s) with rc (%s) rootRc (%s)", + index->GetName().c_str(), + MOT::RcToString(u_sess->mot_cxt.jit_context->m_rc), + MOT::RcToString(rootRc)); } return itr; @@ -776,7 +1323,17 @@ MOT::IndexIterator* searchIterator(MOT::Index* index, MOT::Key* key, int forward MOT::IndexIterator* beginIterator(MOT::Index* index) { - return index->Begin(MOTCurrThreadId); + MOT::IndexIterator* itr = index->Begin(MOTCurrThreadId); + if (itr == nullptr) { + MOT::RC rootRc = MOT_GET_ROOT_ERROR_RC(); + u_sess->mot_cxt.jit_context->m_rc = rootRc != MOT::RC_OK ? rootRc : MOT::RC_MEMORY_ALLOCATION_ERROR; + MOT_LOG_ERROR("beginIterator(): Failed to create iterator for index (%s) with rc (%s) rootRc (%s)", + index->GetName().c_str(), + MOT::RcToString(u_sess->mot_cxt.jit_context->m_rc), + MOT::RcToString(rootRc)); + } + + return itr; } MOT::IndexIterator* createEndIterator(MOT::Index* index, MOT::Key* key, int forward_scan, int include_bound) @@ -787,8 +1344,8 @@ MOT::IndexIterator* createEndIterator(MOT::Index* index, MOT::Key* key, int forw forward_scan ? false : true; // in forward scan search key or previous, in backwards scan search key or next bool found = false; - MOT_LOG_DEBUG( - "Creating end iterator (forward_scan=%d, forwardDirection=%s, include_bound=%s)) for index %p from key %p: %s", + MOT_LOG_DEBUG("createEndIterator(): Creating end iterator (forward_scan=%d, forwardDirection=%s, " + "include_bound=%s)) for index %p from key %p: %s", forward_scan, forwardDirection ? "true" : "false", include_bound ? "true" : "false", @@ -799,79 +1356,146 @@ MOT::IndexIterator* createEndIterator(MOT::Index* index, MOT::Key* key, int forw MOT_LOG_DEBUG("Current txn is: %p", curr_txn); itr = index->Search(key, matchKey, forwardDirection, curr_txn->GetThdId(), found); - if (!found) { - MOT_LOG_DEBUG("createEndIterator: Exact match not found, still continuing, itr=%p", itr); + if (itr != nullptr) { + if (!found) { + MOT_LOG_DEBUG("createEndIterator(): Exact match not found, still continuing, itr=%p", itr); + } else { + MOT_LOG_DEBUG("createEndIterator(): Exact match found with iterator %p", itr); + } } else { - MOT_LOG_DEBUG("createEndIterator: Exact match found with iterator %p", itr); + MOT::RC rootRc = MOT_GET_ROOT_ERROR_RC(); + u_sess->mot_cxt.jit_context->m_rc = rootRc != MOT::RC_OK ? rootRc : MOT::RC_MEMORY_ALLOCATION_ERROR; + MOT_LOG_ERROR("createEndIterator(): Failed to create iterator for index (%s) with rc (%s) rootRc (%s)", + index->GetName().c_str(), + MOT::RcToString(u_sess->mot_cxt.jit_context->m_rc), + MOT::RcToString(rootRc)); } + return itr; } int isScanEnd(MOT::Index* index, MOT::IndexIterator* itr, MOT::IndexIterator* end_itr, int forward_scan) { - MOT_LOG_DEBUG("Checking if scan ended"); + MOT_LOG_DEBUG("isScanEnd(): Checking if scan ended"); + MOT_ASSERT(itr != nullptr); + if (!itr->IsValid()) { + MOT_LOG_DEBUG("isScanEnd(): begin iterator %p is not valid", itr); + return 1; + } + + // in case of full-scan end iterator is null, and then we can definitely conclude that scan has not ended yet + // (since begin iterator is still valid) + if (end_itr == nullptr) { +#ifdef MOT_JIT_FULL_SCAN + MOT_LOG_DEBUG("isScanEnd(): full-scan not ended yet"); + return 0; +#else + MOT_LOG_TRACE("isScanEnd(): end iterator is null. index %p", index); + u_sess->mot_cxt.jit_context->m_rc = MOT::RC_MEMORY_ALLOCATION_ERROR; + return 1; +#endif + } + + // check end iterator validity + if (!end_itr->IsValid()) { + MOT_LOG_DEBUG("isScanEnd(): end iterator %p is not valid", end_itr); + return 1; + } + + // compare begin/end iterator keys of iterated rows + const MOT::Key* startKey = reinterpret_cast(const_cast(itr->GetKey())); + const MOT::Key* endKey = reinterpret_cast(const_cast(end_itr->GetKey())); + if ((startKey == nullptr) || (endKey == nullptr)) { + MOT_LOG_DEBUG("isScanEnd(): either start key or end key is invalid"); + return 1; + } + MOT_LOG_DEBUG("isScanEnd(): Start key: %s", MOT::HexStr(startKey->GetKeyBuf(), startKey->GetKeyLength()).c_str()); + MOT_LOG_DEBUG("isScanEnd(): End key: %s", MOT::HexStr(endKey->GetKeyBuf(), endKey->GetKeyLength()).c_str()); int res = 0; - - if (itr != nullptr && !itr->IsValid()) { - MOT_LOG_DEBUG("isScanEnd(): begin iterator is not valid"); - res = 1; - } else if (end_itr != nullptr && !end_itr->IsValid()) { - MOT_LOG_DEBUG("isScanEnd(): end iterator is not valid"); - res = 1; + int cmpRes = memcmp(startKey->GetKeyBuf(), endKey->GetKeyBuf(), index->GetKeySizeNoSuffix()); + MOT_LOG_DEBUG("isScanEnd(): cmpRes = %d", cmpRes); + if (forward_scan) { + if (cmpRes > 0) { // end key is included in scan (so == is not reported as end of scan) + MOT_LOG_DEBUG("isScanEnd(): end of forward scan detected"); + res = 1; + } } else { - const MOT::Key* startKey = nullptr; - const MOT::Key* endKey = nullptr; - - if (itr != nullptr) { - startKey = reinterpret_cast(const_cast(itr->GetKey())); - MOT_LOG_DEBUG("Start key: %s", MOT::HexStr(startKey->GetKeyBuf(), startKey->GetKeyLength()).c_str()); - } - if (end_itr != nullptr) { - endKey = reinterpret_cast(const_cast(end_itr->GetKey())); - MOT_LOG_DEBUG("End key: %s", MOT::HexStr(endKey->GetKeyBuf(), endKey->GetKeyLength()).c_str()); - } - - if (startKey != nullptr && endKey != nullptr) { - int cmpRes = memcmp(startKey->GetKeyBuf(), endKey->GetKeyBuf(), index->GetKeySizeNoSuffix()); - MOT_LOG_DEBUG("isScanEnd(): cmpRes = %d", cmpRes); - if (forward_scan) { - if (cmpRes > 0) { // end key is included in scan (so == is not reported as end of scan) - MOT_LOG_DEBUG("isScanEnd(): end of forward scan detected"); - res = 1; - } - } else { - if (cmpRes < 0) { - MOT_LOG_DEBUG("isScanEnd(): end of backward scan detected"); - res = 1; - } - } - } else { - MOT_LOG_DEBUG("isScanEnd(): either start key or end key is invalid"); + if (cmpRes < 0) { + MOT_LOG_DEBUG("isScanEnd(): end of backward scan detected"); + res = 1; } } return res; } -MOT::Row* getRowFromIterator( - MOT::Index* index, MOT::IndexIterator* itr, MOT::IndexIterator* end_itr, int access_mode, int forward_scan) +int CheckRowExistsInIterator(MOT::Index* index, MOT::IndexIterator* itr, MOT::IndexIterator* endItr, int forwardScan) +{ + bool rowExists = false; + MOT::RC rc = MOT::RC_OK; + + MOT_LOG_DEBUG("CheckRowExistsInIterator(): Retrieving row from iterator %p", itr); + MOT::TxnManager* currTxn = u_sess->mot_cxt.jit_txn; + do { + // get row from iterator using primary sentinel + MOT::Sentinel* sentinel = itr->GetPrimarySentinel(); + rowExists = currTxn->IsRowExist(sentinel, rc); + if (!rowExists) { + if (rc != MOT::RC_OK) { + u_sess->mot_cxt.jit_context->m_rc = rc; + itr->Invalidate(); + break; + } + MOT_LOG_DEBUG("CheckRowExistsInIterator(): Encountered non-existent row during scan, advancing iterator"); + itr->Next(); + continue; + } + + MOT_ASSERT(rc == MOT::RC_OK); + // verify the scan did not pass the end iterator + if (isScanEnd(index, itr, endItr, forwardScan)) { + MOT_LOG_DEBUG("CheckRowExistsInIterator(): Detected end of scan"); + rowExists = false; + itr->Invalidate(); + break; + } + + // prepare already for next round + itr->Next(); + break; + } while (itr->IsValid()); + + MOT_LOG_DEBUG( + "CheckRowExistsInIterator(): Retrieved row %s from iterator %p", rowExists ? "exists" : "not-exist", itr); + return rowExists ? 1 : 0; +} + +MOT::Row* getRowFromIterator(MOT::Index* index, MOT::IndexIterator* itr, MOT::IndexIterator* end_itr, int access_mode, + int forward_scan, int innerRow, int subQueryIndex) { MOT::Row* row = NULL; + MOT::TxnManager* curr_txn = u_sess->mot_cxt.jit_txn; + MOT::RC rc = MOT::RC_OK; MOT_LOG_DEBUG("getRowFromIterator(): Retrieving row from iterator %p", itr); - MOT::TxnManager* curr_txn = u_sess->mot_cxt.jit_txn; do { // get row from iterator using primary sentinel MOT::Sentinel* sentinel = itr->GetPrimarySentinel(); row = curr_txn->RowLookup((MOT::AccessType)access_mode, sentinel, rc); if (row == NULL) { + if (rc != MOT::RC_OK) { + u_sess->mot_cxt.jit_context->m_rc = rc; + itr->Invalidate(); + break; + } MOT_LOG_DEBUG("getRowFromIterator(): Encountered NULL row during scan, advancing iterator"); itr->Next(); continue; } + MOT_ASSERT(rc == MOT::RC_OK); // verify the scan did not pass the end iterator if (isScanEnd(index, itr, end_itr, forward_scan)) { MOT_LOG_DEBUG("getRowFromIterator(): Detected end of scan"); @@ -885,6 +1509,18 @@ MOT::Row* getRowFromIterator( break; } while (itr->IsValid()); + if (row) { + if (access_mode == MOT::AccessType::WR) { + rc = curr_txn->UpdateLastRowState((MOT::AccessType)access_mode); + if (rc != MOT::RC_OK) { + u_sess->mot_cxt.jit_context->m_rc = rc; + row = nullptr; + itr->Invalidate(); + } else { + row = curr_txn->GetLastAccessedDraft(); + } + } + } MOT_LOG_DEBUG("getRowFromIterator(): Retrieved row %p from iterator %p", row, itr); return row; } @@ -892,9 +1528,11 @@ MOT::Row* getRowFromIterator( void destroyIterator(MOT::IndexIterator* itr) { MOT_LOG_DEBUG("Destroying iterator %p", itr); - itr->Invalidate(); - itr->Destroy(); - delete itr; + if (itr != nullptr) { + itr->Invalidate(); + itr->Destroy(); + delete itr; + } } /*--------------------------- Stateful Execution Helpers ---------------------------*/ @@ -902,17 +1540,18 @@ void setStateIterator(MOT::IndexIterator* itr, int begin_itr, int inner_scan) { MOT_LOG_DEBUG("Setting state iterator %p (begin_itr=%d, inner_scan=%d)", itr, begin_itr, inner_scan); if (u_sess->mot_cxt.jit_context) { + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { if (begin_itr) { - u_sess->mot_cxt.jit_context->m_innerBeginIterator = itr; + execState->m_innerBeginIterator = itr; } else { - u_sess->mot_cxt.jit_context->m_innerEndIterator = itr; + execState->m_innerEndIterator = itr; } } else { if (begin_itr) { - u_sess->mot_cxt.jit_context->m_beginIterator = itr; + execState->m_beginIterator = itr; } else { - u_sess->mot_cxt.jit_context->m_endIterator = itr; + execState->m_endIterator = itr; } } } @@ -922,17 +1561,18 @@ MOT::IndexIterator* getStateIterator(int begin_itr, int inner_scan) { MOT::IndexIterator* result = NULL; if (u_sess->mot_cxt.jit_context) { + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { if (begin_itr) { - result = u_sess->mot_cxt.jit_context->m_innerBeginIterator; + result = execState->m_innerBeginIterator; } else { - result = u_sess->mot_cxt.jit_context->m_innerEndIterator; + result = execState->m_innerEndIterator; } } else { if (begin_itr) { - result = u_sess->mot_cxt.jit_context->m_beginIterator; + result = execState->m_beginIterator; } else { - result = u_sess->mot_cxt.jit_context->m_endIterator; + result = execState->m_endIterator; } } } @@ -944,17 +1584,18 @@ int isStateIteratorNull(int begin_itr, int inner_scan) { int is_null = 0; if (u_sess->mot_cxt.jit_context) { + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { if (begin_itr) { - is_null = u_sess->mot_cxt.jit_context->m_innerBeginIterator ? 0 : 1; + is_null = execState->m_innerBeginIterator ? 0 : 1; } else { - is_null = u_sess->mot_cxt.jit_context->m_innerEndIterator ? 0 : 1; + is_null = execState->m_innerEndIterator ? 0 : 1; } } else { if (begin_itr) { - is_null = u_sess->mot_cxt.jit_context->m_beginIterator ? 0 : 1; + is_null = execState->m_beginIterator ? 0 : 1; } else { - is_null = u_sess->mot_cxt.jit_context->m_endIterator ? 0 : 1; + is_null = execState->m_endIterator ? 0 : 1; } } } @@ -966,16 +1607,13 @@ int isStateIteratorNull(int begin_itr, int inner_scan) int isStateScanEnd(int forward_scan, int inner_scan) { int result = -1; + JitExec::JitQueryContext* jitContext = (JitExec::JitQueryContext*)u_sess->mot_cxt.jit_context; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - result = isScanEnd(u_sess->mot_cxt.jit_context->m_innerIndex, - u_sess->mot_cxt.jit_context->m_innerBeginIterator, - u_sess->mot_cxt.jit_context->m_innerEndIterator, - forward_scan); + result = isScanEnd( + jitContext->m_innerIndex, execState->m_innerBeginIterator, execState->m_innerEndIterator, forward_scan); } else { - result = isScanEnd(u_sess->mot_cxt.jit_context->m_index, - u_sess->mot_cxt.jit_context->m_beginIterator, - u_sess->mot_cxt.jit_context->m_endIterator, - forward_scan); + result = isScanEnd(jitContext->m_index, execState->m_beginIterator, execState->m_endIterator, forward_scan); } MOT_LOG_DEBUG( "Checked if state scan ended (forward_scan=%d, inner_scan=%d): result=%d", forward_scan, inner_scan, result); @@ -985,18 +1623,24 @@ int isStateScanEnd(int forward_scan, int inner_scan) MOT::Row* getRowFromStateIterator(int access_mode, int forward_scan, int inner_scan) { MOT::Row* result = NULL; + JitExec::JitQueryContext* jitContext = (JitExec::JitQueryContext*)u_sess->mot_cxt.jit_context; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - result = getRowFromIterator(u_sess->mot_cxt.jit_context->m_innerIndex, - u_sess->mot_cxt.jit_context->m_innerBeginIterator, - u_sess->mot_cxt.jit_context->m_innerEndIterator, + result = getRowFromIterator(jitContext->m_innerIndex, + execState->m_innerBeginIterator, + execState->m_innerEndIterator, access_mode, - forward_scan); + forward_scan, + inner_scan, + -1); } else { - result = getRowFromIterator(u_sess->mot_cxt.jit_context->m_index, - u_sess->mot_cxt.jit_context->m_beginIterator, - u_sess->mot_cxt.jit_context->m_endIterator, + result = getRowFromIterator(jitContext->m_index, + execState->m_beginIterator, + execState->m_endIterator, access_mode, - forward_scan); + forward_scan, + inner_scan, + -1); } MOT_LOG_DEBUG("Retrieved row %p from state iterator (access_mode=%d, forward_scan=%d, inner_scan=%d): result=%d", result, @@ -1009,23 +1653,24 @@ MOT::Row* getRowFromStateIterator(int access_mode, int forward_scan, int inner_s void destroyStateIterators(int inner_scan) { MOT_LOG_DEBUG("Destroying state iterators (inner_scan=%d)", inner_scan); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - if (u_sess->mot_cxt.jit_context->m_innerBeginIterator) { - destroyIterator(u_sess->mot_cxt.jit_context->m_innerBeginIterator); - u_sess->mot_cxt.jit_context->m_innerBeginIterator = NULL; + if (execState->m_innerBeginIterator) { + destroyIterator(execState->m_innerBeginIterator); + execState->m_innerBeginIterator = NULL; } - if (u_sess->mot_cxt.jit_context->m_innerEndIterator) { - destroyIterator(u_sess->mot_cxt.jit_context->m_innerEndIterator); - u_sess->mot_cxt.jit_context->m_innerEndIterator = NULL; + if (execState->m_innerEndIterator) { + destroyIterator(execState->m_innerEndIterator); + execState->m_innerEndIterator = NULL; } } else { - if (u_sess->mot_cxt.jit_context->m_beginIterator) { - destroyIterator(u_sess->mot_cxt.jit_context->m_beginIterator); - u_sess->mot_cxt.jit_context->m_beginIterator = NULL; + if (execState->m_beginIterator) { + destroyIterator(execState->m_beginIterator); + execState->m_beginIterator = NULL; } - if (u_sess->mot_cxt.jit_context->m_endIterator) { - destroyIterator(u_sess->mot_cxt.jit_context->m_endIterator); - u_sess->mot_cxt.jit_context->m_endIterator = NULL; + if (execState->m_endIterator) { + destroyIterator(execState->m_endIterator); + execState->m_endIterator = NULL; } } } @@ -1033,20 +1678,22 @@ void destroyStateIterators(int inner_scan) void setStateScanEndFlag(int scan_ended, int inner_scan) { MOT_LOG_DEBUG("Setting state scan end flag to %d (inner_scan=%d)", scan_ended, inner_scan); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - u_sess->mot_cxt.jit_context->m_innerScanEnded = scan_ended; + execState->m_innerScanEnded = scan_ended; } else { - u_sess->mot_cxt.jit_context->m_scanEnded = scan_ended; + execState->m_scanEnded = scan_ended; } } int getStateScanEndFlag(int inner_scan) { int result = -1; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - result = (int)u_sess->mot_cxt.jit_context->m_innerScanEnded; + result = (int)execState->m_innerScanEnded; } else { - result = (int)u_sess->mot_cxt.jit_context->m_scanEnded; + result = (int)execState->m_scanEnded; } MOT_LOG_DEBUG("Retrieved state scan end flag %d (inner_scan=%d)", result, inner_scan); return result; @@ -1055,30 +1702,33 @@ int getStateScanEndFlag(int inner_scan) void resetStateRow(int inner_scan) { MOT_LOG_DEBUG("Resetting state row to NULL (inner_scan=%d)", inner_scan); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - u_sess->mot_cxt.jit_context->m_innerRow = NULL; + execState->m_innerRow = NULL; } else { - u_sess->mot_cxt.jit_context->m_row = NULL; + execState->m_row = NULL; } } void setStateRow(MOT::Row* row, int inner_scan) { MOT_LOG_DEBUG("Setting state row to %p (inner_scan=%d)", row, inner_scan); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - u_sess->mot_cxt.jit_context->m_innerRow = row; + execState->m_innerRow = row; } else { - u_sess->mot_cxt.jit_context->m_row = row; + execState->m_row = row; } } MOT::Row* getStateRow(int inner_scan) { MOT::Row* result = NULL; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - result = u_sess->mot_cxt.jit_context->m_innerRow; + result = execState->m_innerRow; } else { - result = u_sess->mot_cxt.jit_context->m_row; + result = execState->m_row; } MOT_LOG_DEBUG("Retrieved state row %p (inner_scan=%d)", result, inner_scan); return result; @@ -1086,25 +1736,27 @@ MOT::Row* getStateRow(int inner_scan) void copyOuterStateRow() { - MOT_LOG_DEBUG("Copying outer state row %p into safe copy %p (for JOIN query)", - u_sess->mot_cxt.jit_context->m_row, - u_sess->mot_cxt.jit_context->m_outerRowCopy); - u_sess->mot_cxt.jit_context->m_outerRowCopy->Copy(u_sess->mot_cxt.jit_context->m_row); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + MOT_LOG_DEBUG( + "Copying outer state row %p into safe copy %p (for JOIN query)", execState->m_row, execState->m_outerRowCopy); + execState->m_outerRowCopy->Copy(execState->m_row); } MOT::Row* getOuterStateRowCopy() { - MOT_LOG_DEBUG("Retrieved outer state row copy %p (for JOIN query)", u_sess->mot_cxt.jit_context->m_outerRowCopy); - return u_sess->mot_cxt.jit_context->m_outerRowCopy; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + MOT_LOG_DEBUG("Retrieved outer state row copy %p (for JOIN query)", execState->m_outerRowCopy); + return execState->m_outerRowCopy; } int isStateRowNull(int inner_scan) { int result = -1; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; if (inner_scan) { - result = (u_sess->mot_cxt.jit_context->m_innerRow == NULL) ? 1 : 0; + result = (execState->m_innerRow == NULL) ? 1 : 0; } else { - result = (u_sess->mot_cxt.jit_context->m_row == NULL) ? 1 : 0; + result = (execState->m_row == NULL) ? 1 : 0; } MOT_LOG_DEBUG("Checked if state row is null (inner_scan=%d): result=%d", inner_scan, result); return result; @@ -1112,190 +1764,307 @@ int isStateRowNull(int inner_scan) void resetStateLimitCounter() { + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; MOT_LOG_DEBUG("Resetting state limit counter to 0"); - u_sess->mot_cxt.jit_context->m_limitCounter = 0; + execState->m_limitCounter = 0; } void incrementStateLimitCounter() { - ++u_sess->mot_cxt.jit_context->m_limitCounter; - MOT_LOG_DEBUG("Incremented state limit counter to %d", u_sess->mot_cxt.jit_context->m_limitCounter); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + ++execState->m_limitCounter; + MOT_LOG_DEBUG("Incremented state limit counter to %u", execState->m_limitCounter); } int getStateLimitCounter() { - return u_sess->mot_cxt.jit_context->m_limitCounter; - MOT_LOG_DEBUG("Retrieved state limit counter with value %d", u_sess->mot_cxt.jit_context->m_limitCounter); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + MOT_LOG_DEBUG("Retrieved state limit counter with value %u", execState->m_limitCounter); + return execState->m_limitCounter; } -void prepareAvgArray(int element_type, int element_count) +void DebugPrintNumericArray(Datum arrayDatum) { + ArrayType* avgArray = (ArrayType*)DatumGetPointer(arrayDatum); + int elmlen = -1; + int elmbyval = false; + char elmalign = 'i'; + bool isNull; + int idx = 0; // 1-based index + Datum value = array_ref(avgArray, 1, &idx, 0, elmlen, elmbyval, elmalign, &isNull); + MOT_LOG_DEBUG("AVG Array[0] at %p", value); + DEBUG_PRINT_DATUM("AVG Array[0]", avgArray->elemtype, value, isNull); + idx = 1; + value = array_ref(avgArray, 1, &idx, 0, elmlen, elmbyval, elmalign, &isNull); + MOT_LOG_DEBUG("AVG Array[1] at %p", value); + DEBUG_PRINT_DATUM("AVG Array[1]", avgArray->elemtype, value, isNull); +} + +void prepareAvgArray(int aggIndex, int element_type, int element_count) +{ + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; MOT_LOG_DEBUG("Preparing AVG() array with %d elements of type %d", element_count, element_type); Datum* elements = (Datum*)palloc(sizeof(Datum) * element_count); for (int i = 0; i < element_count; ++i) { if (element_type == NUMERICOID) { elements[i] = makeNumericZero(); + DEBUG_PRINT_DATUM("AVG initial Element", NUMERICOID, elements[i], false); } else if (element_type == INT8OID) { elements[i] = Int64GetDatum(0); } else if (element_type == FLOAT8OID) { elements[i] = Float8GetDatum(0.0f); } + MOT_LOG_DEBUG("AVG Element %d: %p", i, elements[i]); } - int elmlen = 0; + // initialize values for int8 and float8 + bool elmbyval = true; // int8, sometimes also float8 (depending on compile flags) + int elmlen = 8; + char elmalign = 'd'; if (element_type == NUMERICOID) { elmlen = -1; // Numeric is a varlena object (see definition of NumericData at utils/numeric.h, and numeric type // in catalog/pg_type.h) - } else if (element_type == INT8OID || element_type == FLOAT8OID) { - elmlen = 8; + elmbyval = false; + elmalign = 'i'; + } else if (element_type == FLOAT8OID) { + elmbyval = FLOAT8PASSBYVAL; } - ArrayType* avg_array = construct_array(elements, element_count, element_type, elmlen, true, 0); - u_sess->mot_cxt.jit_context->m_avgArray = PointerGetDatum(avg_array); + ArrayType* avg_array = construct_array(elements, element_count, element_type, elmlen, elmbyval, elmalign); + execState->m_avgArray = PointerGetDatum(avg_array); + execState->m_aggValueIsNull = 1; +#ifdef MOT_JIT_DEBUG + MOT_LOG_DEBUG("AVG array at %p", execState->m_avgArray); + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { + DebugPrintNumericArray(execState->m_avgArray); + } +#endif } -Datum loadAvgArray() +Datum loadAvgArray(int aggIndex) { - Datum result = u_sess->mot_cxt.jit_context->m_avgArray; - MOT_LOG_DEBUG("Loaded AVG() array %" PRIu64, (uint64_t)result); + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; + Datum result = execState->m_avgArray; + MOT_LOG_DEBUG("Loaded AVG() array at %p", result); +#ifdef MOT_JIT_DEBUG + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { + DebugPrintNumericArray(result); + } +#endif return result; } -void saveAvgArray(Datum avg_array) +void saveAvgArray(int aggIndex, Datum avg_array) { - MOT_LOG_DEBUG("Saving AVG() array %" PRIu64, (uint64_t)avg_array); - u_sess->mot_cxt.jit_context->m_avgArray = avg_array; + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; + MOT_LOG_DEBUG("Saving AVG() array %p", avg_array); +#ifdef MOT_JIT_DEBUG + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { + DebugPrintNumericArray(avg_array); + } +#endif + execState->m_avgArray = avg_array; + execState->m_aggValueIsNull = 0; } -Datum computeAvgFromArray(int element_type) +Datum ComputeAvg(PGFunction func, Datum avgArray, bool* isNull) { + FunctionCallInfoData fcinfo; + Datum result; + + InitFunctionCallInfoData(fcinfo, NULL, 1, InvalidOid, NULL, NULL); + + fcinfo.arg[0] = avgArray; + fcinfo.argnull[0] = false; + + result = (*func)(&fcinfo); + + // communicate back is-null status + *isNull = fcinfo.isnull; + + return result; +} + +Datum computeAvgFromArray(int aggIndex, int element_type) +{ + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; + + MOT_LOG_DEBUG("Computing AVG() array %" PRIu64, (uint64_t)execState->m_avgArray); +#ifdef MOT_JIT_DEBUG + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { + DebugPrintNumericArray(execState->m_avgArray); + } +#endif + + // attention: in case of null result an ereport exception is thrown, so we call avg functions carefully + bool isNull = false; Datum avg = PointerGetDatum(NULL); if (element_type == NUMERICOID) { - avg = DirectFunctionCall1(numeric_avg, u_sess->mot_cxt.jit_context->m_avgArray); + avg = ComputeAvg(numeric_avg, execState->m_avgArray, &isNull); } else if (element_type == INT8OID) { - avg = DirectFunctionCall1(int8_avg, u_sess->mot_cxt.jit_context->m_avgArray); + avg = ComputeAvg(int8_avg, execState->m_avgArray, &isNull); } else if (element_type == FLOAT8OID) { - avg = DirectFunctionCall1(float8_avg, u_sess->mot_cxt.jit_context->m_avgArray); + avg = ComputeAvg(float8_avg, execState->m_avgArray, &isNull); } - MOT_LOG_DEBUG("Computed AVG() from array %" PRIu64 ": %f", - (uint64_t)u_sess->mot_cxt.jit_context->m_avgArray, - numericToDouble(avg)); + + SET_EXPR_IS_NULL(isNull); return avg; } -void resetAggValue(int element_type) +void resetAggValue(int aggIndex, int element_type) { + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; MOT_LOG_DEBUG("Resetting aggregated value to zero of type %d", element_type); if (element_type == NUMERICOID) { - u_sess->mot_cxt.jit_context->m_aggValue = makeNumericZero(); + execState->m_aggValue = makeNumericZero(); } else if (element_type == INT8OID) { - u_sess->mot_cxt.jit_context->m_aggValue = Int64GetDatum(0); + execState->m_aggValue = Int64GetDatum(0); } else if (element_type == FLOAT4OID) { - u_sess->mot_cxt.jit_context->m_aggValue = Float4GetDatum(((float)0.0)); + execState->m_aggValue = Float4GetDatum(((float)0.0)); } else if (element_type == FLOAT8OID) { - u_sess->mot_cxt.jit_context->m_aggValue = Float8GetDatum(((double)0.0)); + execState->m_aggValue = Float8GetDatum(((double)0.0)); } + execState->m_aggValueIsNull = 1; } -void resetCountAgg() +Datum getAggValue(int aggIndex) { - MOT_LOG_DEBUG("Resetting aggregated count value to 0"); - u_sess->mot_cxt.jit_context->m_aggValue = Int64GetDatum(0); -} - -Datum getAggValue() -{ - Datum result = u_sess->mot_cxt.jit_context->m_aggValue; + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; + Datum result = execState->m_aggValue; MOT_LOG_DEBUG("Retrieved aggregated value: %" PRIu64, result); return result; } -void setAggValue(Datum value) +void setAggValue(int aggIndex, Datum value) { + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; MOT_LOG_DEBUG("Setting aggregated value to: %" PRIu64, value); - u_sess->mot_cxt.jit_context->m_aggValue = value; + execState->m_aggValue = value; + execState->m_aggValueIsNull = 0; } -void resetAggMaxMinNull() +int getAggValueIsNull(int aggIndex) { - MOT_LOG_DEBUG("Resetting aggregated max/min null flag to true"); - u_sess->mot_cxt.jit_context->m_maxMinAggNull = 1; -} - -void setAggMaxMinNotNull() -{ - MOT_LOG_DEBUG("Setting aggregated max/min null flag to false"); - u_sess->mot_cxt.jit_context->m_maxMinAggNull = 0; -} - -int getAggMaxMinIsNull() -{ - int result = (int)u_sess->mot_cxt.jit_context->m_maxMinAggNull; - MOT_LOG_DEBUG("Retrieved aggregated max/min null flag: %d", result); + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; + int result = execState->m_aggValueIsNull; + MOT_LOG_DEBUG("Retrieved aggregated value is-null: %d", result); return result; } -void prepareDistinctSet(int element_type) +int setAggValueIsNull(int aggIndex, int isNull) { + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; + execState->m_aggValueIsNull = isNull; + MOT_LOG_DEBUG("Set aggregated value is-null to: %d", isNull); + return 0; +} + +void prepareDistinctSet(int aggIndex, int element_type) +{ + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; switch (element_type) { case INT8OID: case INT4OID: case INT2OID: case INT1OID: MOT_LOG_DEBUG("Preparing distinct integer set for type %d", element_type); - u_sess->mot_cxt.jit_context->m_distinctSet = prepareDistinctIntSet(); + execState->m_distinctSet = prepareDistinctIntSet(); break; case FLOAT4OID: case FLOAT8OID: - case NUMERICOID: MOT_LOG_DEBUG("Preparing distinct double-precision value set for type %d", element_type); - u_sess->mot_cxt.jit_context->m_distinctSet = prepareDistinctDoubleSet(); + execState->m_distinctSet = prepareDistinctDoubleSet(); break; + + case NUMERICOID: + MOT_LOG_DEBUG("Preparing distinct numeric value set for type %d", element_type); + execState->m_distinctSet = prepareDistinctNumericSet(); + break; + + case VARCHAROID: + MOT_LOG_DEBUG("Preparing distinct varchar value set for type %d", element_type); + execState->m_distinctSet = prepareDistinctVarcharSet(); + break; + default: MOT_LOG_ERROR("Input element type %d is invalid", element_type); break; } + + if (execState->m_distinctSet == nullptr) { + MOT_LOG_ERROR("Failed to create distinctSet for element type %d", element_type); + RaiseResourceLimitFault(); + } } -int insertDistinctItem(int element_type, Datum item) +int insertDistinctItem(int aggIndex, int element_type, Datum item) { + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; int result = 0; switch (element_type) { case INT8OID: MOT_LOG_DEBUG("Inserting distinct int8 value %" PRIu64, DatumGetInt64(item)); - result = insertDistinctIntItem(u_sess->mot_cxt.jit_context->m_distinctSet, DatumGetInt64(item)); + result = insertDistinctIntItem(execState->m_distinctSet, DatumGetInt64(item)); break; case INT4OID: MOT_LOG_DEBUG("Inserting distinct int4 value %" PRIu64, DatumGetInt32(item)); - result = insertDistinctIntItem(u_sess->mot_cxt.jit_context->m_distinctSet, DatumGetInt32(item)); + result = insertDistinctIntItem(execState->m_distinctSet, DatumGetInt32(item)); break; case INT2OID: MOT_LOG_DEBUG("Inserting distinct int2 value %" PRIu64, DatumGetInt16(item)); - result = insertDistinctIntItem(u_sess->mot_cxt.jit_context->m_distinctSet, DatumGetInt16(item)); + result = insertDistinctIntItem(execState->m_distinctSet, DatumGetInt16(item)); break; case INT1OID: MOT_LOG_DEBUG("Inserting distinct int1 value %" PRIu64, DatumGetInt8(item)); - result = insertDistinctIntItem(u_sess->mot_cxt.jit_context->m_distinctSet, DatumGetInt8(item)); + result = insertDistinctIntItem(execState->m_distinctSet, DatumGetInt8(item)); break; case FLOAT4OID: MOT_LOG_DEBUG("Inserting distinct float4 value %f", (float)DatumGetFloat4(item)); - result = insertDistinctDoubleItem(u_sess->mot_cxt.jit_context->m_distinctSet, DatumGetFloat4(item)); + result = insertDistinctDoubleItem(execState->m_distinctSet, DatumGetFloat4(item)); break; case FLOAT8OID: MOT_LOG_DEBUG("Inserting distinct float8 value %f", (double)DatumGetFloat8(item)); - result = insertDistinctDoubleItem(u_sess->mot_cxt.jit_context->m_distinctSet, DatumGetFloat8(item)); + result = insertDistinctDoubleItem(execState->m_distinctSet, DatumGetFloat8(item)); break; case NUMERICOID: - MOT_LOG_DEBUG("Inserting distinct numeric value %f", numericToDouble(item)); - result = insertDistinctDoubleItem(u_sess->mot_cxt.jit_context->m_distinctSet, numericToDouble(item)); + MOT_LOG_DEBUG("Inserting distinct numeric value %f", JitExec::NumericToDouble(item)); + result = insertDistinctNumericItem(execState->m_distinctSet, item); break; + + case VARCHAROID: + DEBUG_PRINT_DATUM("Inserting distinct varchar value", VARCHAROID, item, false); + result = insertDistinctVarcharItem(execState->m_distinctSet, item); + break; + default: MOT_LOG_ERROR("Input element type %d is invalid", element_type); break; @@ -1303,23 +2072,36 @@ int insertDistinctItem(int element_type, Datum item) return result; } -void destroyDistinctSet(int element_type) +void destroyDistinctSet(int aggIndex, int element_type) { + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + VERIFY_PTR(queryExecState->m_aggExecState, "NULL aggregate array"); + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[aggIndex]; switch (element_type) { case INT8OID: case INT4OID: case INT2OID: case INT1OID: MOT_LOG_DEBUG("Destroying distinct integer set for type %d", element_type); - destroyDistinctIntSet(u_sess->mot_cxt.jit_context->m_distinctSet); + destroyDistinctIntSet(execState->m_distinctSet); break; case FLOAT4OID: case FLOAT8OID: - case NUMERICOID: MOT_LOG_DEBUG("Destroying distinct double-precision value set for type %d", element_type); - destroyDistinctDoubleSet(u_sess->mot_cxt.jit_context->m_distinctSet); + destroyDistinctDoubleSet(execState->m_distinctSet); break; + + case NUMERICOID: + MOT_LOG_DEBUG("Destroying distinct numeric value set for type %d", element_type); + destroyDistinctNumericSet(execState->m_distinctSet); + break; + + case VARCHAROID: + MOT_LOG_DEBUG("Destroying distinct varchar value set for type %d", element_type); + destroyDistinctVarcharSet(execState->m_distinctSet); + break; + default: MOT_LOG_ERROR("Input element type %d is invalid", element_type); break; @@ -1332,27 +2114,27 @@ void resetTupleDatum(TupleTableSlot* slot, int tuple_colid, int zero_type) MOT_LOG_DEBUG("Resetting tuple %p column %d datum to typed zero by type %d", slot, tuple_colid, zero_type) switch (zero_type) { case NUMERICOID: - writeTupleDatum(slot, tuple_colid, makeNumericZero()); + writeTupleDatum(slot, tuple_colid, makeNumericZero(), 1); break; case FLOAT8OID: - writeTupleDatum(slot, tuple_colid, Float8GetDatum(0)); + writeTupleDatum(slot, tuple_colid, Float8GetDatum(0), 1); break; case FLOAT4OID: - writeTupleDatum(slot, tuple_colid, Float4GetDatum(0)); + writeTupleDatum(slot, tuple_colid, Float4GetDatum(0), 1); break; case INT8OID: - writeTupleDatum(slot, tuple_colid, Int64GetDatum(0)); + writeTupleDatum(slot, tuple_colid, Int64GetDatum(0), 1); break; case INT4OID: - writeTupleDatum(slot, tuple_colid, Int32GetDatum(0)); + writeTupleDatum(slot, tuple_colid, Int32GetDatum(0), 1); break; case INT2OID: - writeTupleDatum(slot, tuple_colid, Int16GetDatum(0)); + writeTupleDatum(slot, tuple_colid, Int16GetDatum(0), 1); break; default: @@ -1361,66 +2143,2124 @@ void resetTupleDatum(TupleTableSlot* slot, int tuple_colid, int zero_type) } } -Datum readTupleDatum(TupleTableSlot* slot, int tuple_colid, int arg_pos) +Datum readTupleDatum(TupleTableSlot* slot, int tuple_colid) { MOT_LOG_DEBUG("Reading datum from tuple column %d ", tuple_colid); Datum result = slot->tts_values[tuple_colid]; - DBG_PRINT_DATUM("Pre-sum Tuple Datum", - slot->tts_tupleDescriptor->attrs[tuple_colid]->atttypid, + DEBUG_PRINT_DATUM("Pre-sum Tuple Datum", + slot->tts_tupleDescriptor->attrs[tuple_colid].atttypid, slot->tts_values[tuple_colid], slot->tts_isnull[tuple_colid]); bool isnull = (result == PointerGetDatum(NULL)); - setExprArgIsNull(arg_pos, (int)isnull); + SET_EXPR_IS_NULL((int)isnull); return result; } -void writeTupleDatum(TupleTableSlot* slot, int tuple_colid, Datum datum) +void writeTupleDatum(TupleTableSlot* slot, int tuple_colid, Datum datum, int isnull) { - MOT_LOG_DEBUG("Writing datum to tuple column %d ", tuple_colid); - bool isnull = (datum == PointerGetDatum(NULL)); + MOT_LOG_DEBUG("Writing datum to tuple column %d, isnull %d", tuple_colid, isnull); slot->tts_values[tuple_colid] = datum; slot->tts_isnull[tuple_colid] = isnull; - DBG_PRINT_DATUM("Post-sum Tuple Datum", - slot->tts_tupleDescriptor->attrs[tuple_colid]->atttypid, + DEBUG_PRINT_DATUM("Post-sum Tuple Datum", + slot->tts_tupleDescriptor->attrs[tuple_colid].atttypid, slot->tts_values[tuple_colid], slot->tts_isnull[tuple_colid]); } Datum SelectSubQueryResult(int subQueryIndex) { - JitExec::JitContext* jitContext = u_sess->mot_cxt.jit_context; - return readTupleDatum(jitContext->m_subQueryData[subQueryIndex].m_slot, 0, 0); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + return readTupleDatum(execState->m_subQueryExecState[subQueryIndex].m_slot, 0); } void CopyAggregateToSubQueryResult(int subQueryIndex) { + // sub-query can have only 1 aggregate MOT_LOG_DEBUG("Copying aggregate datum to sub-query %d slot", subQueryIndex); - JitExec::JitContext* jitContext = u_sess->mot_cxt.jit_context; - writeTupleDatum(jitContext->m_subQueryData[subQueryIndex].m_slot, 0, jitContext->m_aggValue); + JitExec::JitQueryExecState* queryExecState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + JitExec::JitAggExecState* execState = &queryExecState->m_aggExecState[0]; + writeTupleDatum(queryExecState->m_subQueryExecState[subQueryIndex].m_slot, + 0, + execState->m_aggValue, + execState->m_aggValueIsNull); } TupleTableSlot* GetSubQuerySlot(int subQueryIndex) { - return u_sess->mot_cxt.jit_context->m_subQueryData[subQueryIndex].m_slot; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + return execState->m_subQueryExecState[subQueryIndex].m_slot; } MOT::Table* GetSubQueryTable(int subQueryIndex) { - return u_sess->mot_cxt.jit_context->m_subQueryData[subQueryIndex].m_table; + JitExec::JitQueryContext* jitContext = (JitExec::JitQueryContext*)u_sess->mot_cxt.jit_context; + return jitContext->m_subQueryContext[subQueryIndex].m_table; } MOT::Index* GetSubQueryIndex(int subQueryIndex) { - return u_sess->mot_cxt.jit_context->m_subQueryData[subQueryIndex].m_index; + JitExec::JitQueryContext* jitContext = (JitExec::JitQueryContext*)u_sess->mot_cxt.jit_context; + return jitContext->m_subQueryContext[subQueryIndex].m_index; } MOT::Key* GetSubQuerySearchKey(int subQueryIndex) { - return u_sess->mot_cxt.jit_context->m_subQueryData[subQueryIndex].m_searchKey; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + return execState->m_subQueryExecState[subQueryIndex].m_searchKey; } MOT::Key* GetSubQueryEndIteratorKey(int subQueryIndex) { - return u_sess->mot_cxt.jit_context->m_subQueryData[subQueryIndex].m_endIteratorKey; + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)u_sess->mot_cxt.jit_context->m_execState; + return execState->m_subQueryExecState[subQueryIndex].m_endIteratorKey; +} + +int8_t* LlvmPushExceptionFrame() +{ + // allocate exception frame + size_t allocSize = sizeof(JitExec::JitFunctionExecState::ExceptionFrame); + JitExec::JitFunctionExecState::ExceptionFrame* exceptionFrame = + (JitExec::JitFunctionExecState::ExceptionFrame*)MOT::MemSessionAlloc(allocSize); + if (exceptionFrame == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "Execute JIT LLVM Stored Procedure", + "Failed to allocate %u bytes for exception frame, while executing %s. Aborting execution.", + (unsigned)allocSize, + u_sess->mot_cxt.jit_context->m_queryString); + RaiseFault(LLVM_FAULT_RESOURCE_LIMIT); + return nullptr; // to avoid compiler warning + } + + // set exception frame attributes and push exception frame + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + uint64_t frameIndex = (execState->m_exceptionStack != nullptr) ? execState->m_exceptionStack->m_frameIndex + 1 : 0; + exceptionFrame->m_frameIndex = frameIndex; + exceptionFrame->m_next = execState->m_exceptionStack; + execState->m_exceptionStack = exceptionFrame; + int8_t* frame = (int8_t*)exceptionFrame->m_jmpBuf; + MOT_LOG_DEBUG("Pushed exception frame #%d %p with jump buf at %p on exec state %p", + (int)frameIndex, + exceptionFrame, + frame, + execState); + return frame; +} + +int8_t* LlvmGetCurrentExceptionFrame() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + JitExec::JitFunctionExecState::ExceptionFrame* exceptionFrame = execState->m_exceptionStack; + if (exceptionFrame == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Execute JIT LLVM Stored Procedure", + "Cannot get current exception frame: exception stack is empty, while executing %s. Aborting execution.", + u_sess->mot_cxt.jit_context->m_queryString); + RaiseFault(LLVM_FAULT_UNHANDLED_EXCEPTION); + } + + MOT_LOG_DEBUG("Getting current exception frame #%d %p with jump buf at %p", + (int)exceptionFrame->m_frameIndex, + exceptionFrame, + (int8_t*)execState->m_exceptionStack->m_jmpBuf); + return (int8_t*)execState->m_exceptionStack->m_jmpBuf; +} + +int LlvmPopExceptionFrame() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + JitExec::JitFunctionExecState::ExceptionFrame* exceptionFrame = execState->m_exceptionStack; + if (exceptionFrame == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Execute JIT LLVM Stored Procedure", + "Cannot pop exception frame: exception stack is empty, while executing %s. Aborting execution.", + u_sess->mot_cxt.jit_context->m_queryString); + RaiseFault(LLVM_FAULT_UNHANDLED_EXCEPTION); + } + + MOT_LOG_DEBUG("Popping exception frame #%d %p with jump buf at %p from exec state %p", + (int)exceptionFrame->m_frameIndex, + exceptionFrame, + (int8_t*)exceptionFrame->m_jmpBuf, + execState); + execState->m_exceptionStack = exceptionFrame->m_next; + MOT::MemSessionFree(exceptionFrame); + return execState->m_exceptionValue; +} + +void LlvmThrowException(int exceptionValue) +{ + // verify there is a handler + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + if (jitContext->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + if (execState->m_exceptionStack == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Execute JIT LLVM Stored Procedure", + "Cannot throw exception: exception stack is empty, while executing %s. Aborting execution.", + u_sess->mot_cxt.jit_context->m_queryString); + RaiseFault(LLVM_FAULT_UNHANDLED_EXCEPTION); + } + + // set exception value and raise exception status + execState->m_exceptionStatus = 1; + execState->m_exceptionValue = exceptionValue; + + // jump to handler + MOT_LOG_DEBUG("Throwing exception, jumping to frame #%d %p with jump buf at %p on exec state %p", + (int)execState->m_exceptionStack->m_frameIndex, + execState->m_exceptionStack, + (int8_t*)execState->m_exceptionStack->m_jmpBuf, + execState); + siglongjmp(execState->m_exceptionStack->m_jmpBuf, execState->m_exceptionValue); + } else { + JitExec::JitExecState* execState = (JitExec::JitExecState*)jitContext->m_execState; + // set exception value and raise exception status + execState->m_exceptionStatus = 1; + execState->m_exceptionValue = exceptionValue; + RaiseFault(LLVM_FAULT_UNHANDLED_EXCEPTION); + } +} + +void LlvmRethrowException() +{ + // verify there is a handler + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + if (execState->m_exceptionStack == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Execute JIT LLVM Stored Procedure", + "Cannot re-throw exception: exception stack is empty, while executing %s. Aborting execution.", + u_sess->mot_cxt.jit_context->m_queryString); + RaiseFault(LLVM_FAULT_UNHANDLED_EXCEPTION); + } + + execState->m_exceptionStatus = 1; // raise again exception status flag + MOT_LOG_DEBUG("Re-throwing exception, jumping to frame #%d %p with jump buf at %p on exec state %p", + (int)execState->m_exceptionStack->m_frameIndex, + execState->m_exceptionStack, + (int8_t*)execState->m_exceptionStack->m_jmpBuf, + execState); + siglongjmp(execState->m_exceptionStack->m_jmpBuf, execState->m_exceptionValue); +} + +int LlvmGetExceptionValue() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + return execState->m_exceptionValue; +} + +int LlvmGetExceptionStatus() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + return execState->m_exceptionStatus; +} + +void LlvmResetExceptionStatus() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + execState->m_exceptionStatus = 0; +} + +void LlvmResetExceptionValue() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + execState->m_exceptionValue = 0; +} + +void LlvmUnwindExceptionFrame() +{ + // same as re-throw + MOT_LOG_DEBUG("Unwinding exception") + LlvmRethrowException(); +} + +void LlvmClearExceptionStack() +{ + MOT_LOG_DEBUG("Clearing exception stack"); + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + while (execState->m_exceptionStack != nullptr) { + JitExec::JitFunctionExecState::ExceptionFrame* exceptionFrame = execState->m_exceptionStack; + MOT_LOG_DEBUG("Popping exception frame #%d %p with jump buf at %p from exec state %p", + (int)exceptionFrame->m_frameIndex, + exceptionFrame, + (int8_t*)exceptionFrame->m_jmpBuf, + execState); + execState->m_exceptionStack = exceptionFrame->m_next; + MOT::MemSessionFree(exceptionFrame); + } +} + +void LLvmPrintFrame(const char* msg, int8_t* frame) +{ + MOT_LOG_DEBUG("%s: %p", msg, frame); +} + +void ValidateParams(ParamListInfo params, int paramId = -1) +{ + if (params == nullptr) { + MOT_LOG_ERROR("Attempt to access null parameter list"); + RaiseAccessViolationFault(); + } + + // if we reached here then parameters must be not null + MOT_ASSERT(params != nullptr); + if (paramId != -1) { + if (paramId >= params->numParams) { + MOT_LOG_ERROR("Attempt to access parameter (%d items) list at %p out of range: %d", + params->numParams, + params, + paramId); + RaiseAccessViolationFault(); + } + } +} + +void JitAbortFunction(int errorCode) +{ + RaiseFault(errorCode); +} + +int JitHasNullParam(ParamListInfo params) +{ + ValidateParams(params); // if null long jump is made + MOT_ASSERT(params != nullptr); + for (int i = 0; i < params->numParams; ++i) { + if (params->params[i].isnull) { + return 1; + } + } + return 0; +} + +int JitGetParamCount(ParamListInfo params) +{ + ValidateParams(params); // if null long jump is made + MOT_ASSERT(params != nullptr); + return params->numParams; +} + +Datum JitGetParamAt(ParamListInfo params, int paramId) +{ + ValidateParams(params, paramId); // if null long jump is made + MOT_ASSERT(params != nullptr); + return params->params[paramId].value; +} + +int JitIsParamNull(ParamListInfo params, int paramId) +{ + ValidateParams(params, paramId); // if null long jump is made + int result = (params->params[paramId].isnull ? 1 : 0); + MOT_LOG_DEBUG("JitIsParamNull(%d) = %s", paramId, result ? "true" : "false"); + return result; +} + +static ParamListInfo GetParamListForParam(ParamListInfo params, int& paramId) +{ + // attention: the parameter list passed into SP has the same size as parameter list passed to the query that + // invoked the SP, so parameter ids are the same + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + if (jitContext->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION) { + JitExec::JitFunctionExecState* execState = (JitExec::JitFunctionExecState*)jitContext->m_execState; + JitExec::JitQueryContext* invokingContext = (JitExec::JitQueryContext*)execState->m_invokingContext; + if (invokingContext->m_invokeParamInfo[paramId].m_mode == JitExec::JitParamMode::JIT_PARAM_DIRECT) { + JitExec::JitQueryExecState* invokingExecState = + (JitExec::JitQueryExecState*)execState->m_invokingContext->m_execState; + params = invokingExecState->m_directParams; + paramId = invokingContext->m_invokeParamInfo[paramId].m_index; + MOT_LOG_DEBUG( + "Using parameters at %p from invoking context for parameter at modified pos %d", params, paramId); + return (ParamListInfo)params; + } + } + MOT_LOG_DEBUG("Using original parameters at %p for parameter %d", params, paramId); + return params; +} + +Datum* JitGetParamAtRef(ParamListInfo params, int paramId) +{ + // we check at the current context whether this parameter is direct passed-as-is or not + params = GetParamListForParam(params, paramId); + ValidateParams(params, paramId); // if null long jump is made + MOT_ASSERT(params != nullptr); + Datum* result = &(params->params[paramId].value); + MOT_LOG_DEBUG("Retrieved param %d ref %p from params %p", paramId, result, params); + DEBUG_PRINT_DATUM("datum value: ", params->params[paramId].ptype, *result, params->params[paramId].isnull); + return result; +} + +bool* JitIsParamNullRef(ParamListInfo params, int paramId) +{ + // we check at the current context with this parameter is direct passed-as-is or not + params = GetParamListForParam(params, paramId); + ValidateParams(params, paramId); // if null long jump is made + MOT_ASSERT(params != nullptr); + bool* result = ¶ms->params[paramId].isnull; + MOT_LOG_DEBUG("Retrieved param %d is-null ref %p from params %p: %d", paramId, result, params, (int)*result); + return result; +} + +ParamListInfo GetInvokeParamListInfo() +{ + MOT_ASSERT(u_sess->mot_cxt.jit_context->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_QUERY); + JitExec::JitQueryContext* jitContext = (JitExec::JitQueryContext*)u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext->m_commandType == JitExec::JIT_COMMAND_INVOKE); + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)jitContext->m_execState; + MOT_LOG_DEBUG("Invoke params are at %p", execState->m_invokeParams); + return execState->m_invokeParams; +} + +void DestroyParamListInfo(ParamListInfo params) +{ + MOT::MemSessionFree(params); +} + +void SetParamValue(ParamListInfo params, int paramId, int paramType, Datum datum, int isNull) +{ + params->params[paramId].value = datum; + params->params[paramId].ptype = paramType; + params->params[paramId].isnull = isNull ? true : false; + DEBUG_PRINT_DATUM("param: ", paramType, datum, params->params[paramId].isnull); +} + +void SetSPSubQueryParamValue(int subQueryId, int id, int type, Datum value, int isNull) +{ + MOT_ASSERT(u_sess->mot_cxt.jit_context->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + JitExec::JitFunctionContext* jitContext = (JitExec::JitFunctionContext*)u_sess->mot_cxt.jit_context; + JitExec::JitFunctionExecState* functionExecState = (JitExec::JitFunctionExecState*)jitContext->m_execState; + + ParamExternData* param = &functionExecState->m_invokedQueryExecState[subQueryId].m_params->params[id]; + param->ptype = type; + param->value = value; + param->isnull = isNull ? true : false; + MOT_LOG_DEBUG("Passed to SP sub-query %d param: ", subQueryId); + DEBUG_PRINT_DATUM("Passed to SP sub-query param", type, value, isNull); +} + +inline void PullUpErrorInfo(JitExec::JitExecState* source, JitExec::JitExecState* target) +{ + target->m_nullColumnId = source->m_nullColumnId; + target->m_nullViolationTable = source->m_nullViolationTable; + target->m_errorMessage = source->m_errorMessage; + target->m_errorDetail = source->m_errorDetail; + target->m_errorHint = source->m_errorHint; + target->m_sqlState = source->m_sqlState; + target->m_sqlStateString = source->m_sqlStateString; +} + +static int InvokeJittedStoredProcedure(JitExec::JitQueryContext* jitContext) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile int res = 0; + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile int spiConnectId = SPI_connectid(); + + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)jitContext->m_execState; + MOT_LOG_DEBUG("Invoking jittable SP %s with %u invoke parameters at %p", + jitContext->m_invokeContext->m_queryString, + jitContext->m_invokeParamCount, + execState->m_invokeParams); + + MOT_ASSERT(jitContext->m_invokeContext->m_execState != nullptr); + jitContext->m_invokeContext->m_execState->m_invokingContext = jitContext; + jitContext->m_execState->m_sqlState = 0; + jitContext->m_invokeContext->m_execState->m_sqlState = 0; + PG_TRY(); + { + res = JitExec::JitExecFunction(jitContext->m_invokeContext, + execState->m_invokeParams, + execState->m_invokeSlot, + execState->m_invokeTuplesProcessed, + execState->m_invokeScanEnded); + MOT_LOG_DEBUG("Finished executing invoked stored procedure %s with result: %d (%s)", + jitContext->m_invokeContext->m_queryString, + res, + MOT::RcToString((MOT::RC)res)); + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + res = HandlePGError(jitContext->m_invokeContext->m_execState, "executing jitted SP", spiConnectId); + signalException = true; + } + PG_END_TRY(); + + // pull-up error info + PullUpErrorInfo(jitContext->m_invokeContext->m_execState, jitContext->m_execState); + + // signal to jitted function that an exception is to be thrown + if (signalException) { + SignalException(jitContext); + } + + return res; +} + +static bool CopyHeapTupleHeader(HeapTupleHeader tupleHeader, TupleTableSlot* slot) +{ + // we fake a tuple for things to work + HeapTupleData tupleData; + HeapTuple tuple = &tupleData; + tuple->t_data = tupleHeader; + + TupleDesc tupDesc = slot->tts_tupleDescriptor; + int tupDescAttrCount = tupDesc->natts; + int tupleAttrCount = PointerIsValid(tupleHeader) ? HeapTupleHeaderGetNatts(tupleHeader, tupDesc) : 0; + int resultIndex = 0; + for (int i = 0; i < tupDescAttrCount; ++i) { + // skip dropped columns in destination + if (tupDesc->attrs[i].attisdropped) { + continue; + } + + bool isNull = true; + Datum value = (Datum)0; + if ((i < tupleAttrCount) && !tupDesc->attrs[i].attisdropped) { + value = SPI_getbinval(tuple, tupDesc, i + 1, &isNull); + if (SPI_result != 0) { + MOT_LOG_TRACE( + "Failed to get datum value from tuple: %s (%u)", SPI_result_code_string(SPI_result), SPI_result); + return false; + } + } else { + continue; // skip dropped columns in source + } + SetSlotValue(slot, resultIndex++, value, isNull); + } + + return true; +} + +static int InvokeUnjittableStoredProcedure(JitExec::JitQueryContext* jitContext) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile int res = 0; + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile Datum* args = nullptr; + volatile int* isnull = nullptr; + volatile Oid* argTypes = nullptr; + volatile int spiConnectId = SPI_connectid(); + + JitExec::JitQueryExecState* execState = (JitExec::JitQueryExecState*)jitContext->m_execState; + MOT_LOG_DEBUG("Invoking unjittable SP %s with %u invoke parameters at %p", + jitContext->m_invokedQueryString, + jitContext->m_invokeParamCount, + execState->m_invokeParams); + + execState->m_sqlState = 0; + PG_TRY(); + { + // prepare parameters + int argCount = (int)jitContext->m_invokeParamCount; + args = (Datum*)palloc(argCount * sizeof(Datum)); + isnull = (int*)palloc(argCount * sizeof(int)); + argTypes = (Oid*)palloc(argCount * sizeof(Oid)); + for (int i = 0; i < argCount; ++i) { + if (jitContext->m_invokeParamInfo[i].m_mode == JitExec::JitParamMode::JIT_PARAM_DIRECT) { + int paramId = jitContext->m_invokeParamInfo[i].m_index; + MOT_LOG_DEBUG("Using direct parameter %d at modified pos %d", i, paramId); + args[i] = execState->m_directParams->params[paramId].value; + isnull[i] = execState->m_directParams->params[paramId].isnull; + argTypes[i] = execState->m_directParams->params[paramId].ptype; + } else { + MOT_LOG_DEBUG("Using passed parameter %d", i); + args[i] = execState->m_invokeParams->params[i].value; + isnull[i] = execState->m_invokeParams->params[i].isnull; + argTypes[i] = execState->m_invokeParams->params[i].ptype; + } + } + + // call plpgsql_call_handler + Datum resDatum = JitInvokePGFunctionNImpl(plpgsql_call_handler, + DEFAULT_COLLATION_OID, + 0, + (Datum*)args, + (int*)isnull, + (Oid*)argTypes, + argCount, + jitContext->m_invokedFunctionOid); + + // now copy datum to result tuple + if (execState->m_invokeSlot->tts_tupleDescriptor->natts > 1) { + // result datum is a tuple already in the caller's memory context, just copy the datum + HeapTupleHeader tupleHeader = (HeapTupleHeader)DatumGetPointer(resDatum); + if (!CopyHeapTupleHeader(tupleHeader, execState->m_invokeSlot)) { + MOT_LOG_TRACE("Failed to copy result slot"); + res = (int)MOT::RC_ERROR; + } + } else if (execState->m_invokeSlot->tts_tupleDescriptor->natts == 1) { + // it is OK to have a boxed tuple here as a single attribute + execState->m_invokeSlot->tts_values[0] = resDatum; + execState->m_invokeSlot->tts_isnull[0] = false; + (void)ExecStoreVirtualTuple(execState->m_invokeSlot); + } + MOT_LOG_DEBUG("Finished executing unjittable invoked stored procedure %s with result: %d (%s)", + jitContext->m_invokedQueryString, + res, + MOT::RcToString((MOT::RC)res)); + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + res = MOT::RC_ERROR; + (void)MemoryContextSwitchTo(origCxt); + res = HandlePGError(jitContext->m_execState, "executing unjittable SP", spiConnectId); + signalException = true; + } + PG_END_TRY(); + + // note: no need to pull-up error info, error info already produced in caller's execution state + if (args != nullptr) { + pfree((void*)args); + } + if (isnull != nullptr) { + pfree((void*)isnull); + } + if (argTypes != nullptr) { + pfree((void*)argTypes); + } + // signal to jitted function that an exception is to be thrown + if (signalException) { + SignalException(jitContext); + } + + return res; +} + +int InvokeStoredProcedure() +{ + JitExec::JitQueryContext* jitContext = (JitExec::JitQueryContext*)u_sess->mot_cxt.jit_context; + MOT_ASSERT(u_sess->mot_cxt.jit_context->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_QUERY); + MOT_ASSERT(jitContext->m_commandType == JitExec::JIT_COMMAND_INVOKE); + + int result = 0; + if (jitContext->m_invokeContext != nullptr) { + result = InvokeJittedStoredProcedure(jitContext); + } else { + result = InvokeUnjittableStoredProcedure(jitContext); + } + return result; +} + +int IsCompositeResult(TupleTableSlot* slot) +{ + int res = 0; + if ((slot->tts_tupleDescriptor->natts == 1) && slot->tts_tupleDescriptor->attrs[0].atttypid == RECORDOID) { + res = 1; + } + MOT_LOG_DEBUG("Result is composite: %d", res); + return res; +} + +Datum* CreateResultDatums() +{ + MOT_ASSERT(u_sess->mot_cxt.jit_context->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + JitExec::JitFunctionContext* jitContext = (JitExec::JitFunctionContext*)u_sess->mot_cxt.jit_context; + int natts = jitContext->m_rowTupDesc->natts; + Datum* dvalues = (Datum*)palloc0(natts * sizeof(Datum)); + MOT_LOG_DEBUG("Created %d datums for heap tuple at %p", natts, dvalues); + return dvalues; +} + +bool* CreateResultNulls() +{ + MOT_ASSERT(u_sess->mot_cxt.jit_context->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + JitExec::JitFunctionContext* jitContext = (JitExec::JitFunctionContext*)u_sess->mot_cxt.jit_context; + int natts = jitContext->m_rowTupDesc->natts; + bool* nulls = (bool*)palloc(natts * sizeof(bool)); + MOT_LOG_DEBUG("Created %d nulls for heap tuple at %p", natts, nulls); + return nulls; +} + +void SetResultValue(Datum* datums, bool* nulls, int index, Datum value, int isNull) +{ + MOT_LOG_DEBUG("Setting heap tuple datum/null pair at index %d", index); + datums[index] = value; + nulls[index] = isNull ? true : false; +} + +Datum CreateResultHeapTuple(Datum* dvalues, bool* nulls) +{ + MOT_ASSERT(u_sess->mot_cxt.jit_context->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + JitExec::JitFunctionContext* jitContext = (JitExec::JitFunctionContext*)u_sess->mot_cxt.jit_context; + + // we do not initialize anything, caller is responsible for that + HeapTuple tuple = heap_form_tuple(jitContext->m_rowTupDesc, dvalues, nulls); + MOT_LOG_DEBUG( + "Created heap tuple %p from datum/null arrays using tuple descriptor %p", tuple, jitContext->m_rowTupDesc); + + pfree_ext(dvalues); + pfree_ext(nulls); + + TupleConversionMap* tupMap = convert_tuples_by_position(jitContext->m_rowTupDesc, + jitContext->m_resultTupDesc, + gettext_noop("returned record type does not match expected record type"), + jitContext->m_functionOid); + + if (tupMap != nullptr) { + tuple = do_convert_tuple(tuple, tupMap); + free_conversion_map(tupMap); + MOT_LOG_DEBUG("Converted heap tuple at %p", tuple); + } + + return PointerGetDatum(tuple); +} + +void SetSlotValue(TupleTableSlot* slot, int tupleColId, Datum value, int isNull) +{ + MOT_LOG_DEBUG("Storing value in tuple column %d", tupleColId); + slot->tts_values[tupleColId] = value; + slot->tts_isnull[tupleColId] = isNull ? true : false; + DEBUG_PRINT_DATUM("Slot Datum", + slot->tts_tupleDescriptor->attrs[tupleColId]->atttypid, + slot->tts_values[tupleColId], + slot->tts_isnull[tupleColId]); +} + +Datum GetSlotValue(TupleTableSlot* slot, int tupleColId) +{ + MOT_LOG_DEBUG("Retrieving datum value in tuple %p column %d", slot, tupleColId); + Datum result = slot->tts_values[tupleColId]; + DEBUG_PRINT_DATUM("Slot Datum", + slot->tts_tupleDescriptor->attrs[tupleColId]->atttypid, + slot->tts_values[tupleColId], + slot->tts_isnull[tupleColId]); + return result; +} + +int GetSlotIsNull(TupleTableSlot* slot, int tupleColId) +{ + MOT_LOG_DEBUG("Retrieving datum is-null in tuple column %d", tupleColId); + int result = slot->tts_isnull[tupleColId] == true ? 1 : 0; + MOT_LOG_DEBUG("Slot Datum is-null: %d", result); + return result; +} + +static void ReleaseOneSubTransaction(bool rollback, uint32_t openSubTxCount) +{ + SubTransactionId subXid = InvalidSubTransactionId; + if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_DEBUG)) { + subXid = ::GetCurrentSubTransactionId(); + } + if (rollback) { + MOT_LOG_DEBUG("Rolling back current sub-transaction #%u: id %" PRIu64, openSubTxCount, (uint64_t)subXid); + ::RollbackAndReleaseCurrentSubTransaction(); + } else { + MOT_LOG_DEBUG("Committing current sub-transaction #%u: id %" PRIu64, openSubTxCount, (uint64_t)subXid); + // if current txn is already in aborted unrecoverable state then report error + if (u_sess->mot_cxt.jit_txn->IsTxnAborted()) { + raiseAbortTxnError(); + } else { + ::ReleaseCurrentSubTransaction(); + } + } +} + +SubTransactionId JitGetCurrentSubTransactionId() +{ + return ::GetCurrentSubTransactionId(); +} + +void JitReleaseAllSubTransactions(bool rollback, SubTransactionId untilSubXid) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile int spiConnectId = SPI_connectid(); + + if (rollback) { + MOT_LOG_DEBUG("Rolling-back %u open sub-transactions", execState->m_openSubTxCount); + } else { + MOT_LOG_DEBUG("Committing %u open sub-transactions", execState->m_openSubTxCount); + } + + PG_TRY(); + { + SubTransactionId subXid = ::GetCurrentSubTransactionId(); + while ((execState->m_openSubTxCount > 0) && (subXid != untilSubXid)) { + ReleaseOneSubTransaction(rollback, execState->m_openSubTxCount); + --execState->m_openSubTxCount; + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + const char* action = rollback ? "rolling back all sub-transactions" : "committing all sub-transactions"; + (void)MemoryContextSwitchTo(origCxt); + (void)HandlePGError(jitContext->m_execState, action, spiConnectId); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + if (jitContext->m_execState->m_sqlState == 0) { + // note: we could use a more informative error code + jitContext->m_execState->m_sqlState = ERRCODE_FDW_ERROR; + } + SignalException((JitExec::MotJitContext*)jitContext); + } + execState->m_openSubTxCount = 0; // must be set to zero even if encountered error +} + +void JitBeginBlockWithExceptions() +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + + if (execState->m_spiBlockId >= MOT_JIT_MAX_BLOCK_DEPTH) { + MOT_LOG_ERROR("Cannot push SPI block, reached maximum allowed %d", (int)MOT_JIT_MAX_BLOCK_DEPTH); + RaiseResourceLimitFault(); + } + + MOT_LOG_DEBUG("Entering statement block with exceptions, with SPI nesting %d", execState->m_spiBlockId); + PG_TRY(); + { + if (u_sess->mot_cxt.jit_txn->IsTxnAborted()) { + raiseAbortTxnError(); + } + int connectid = SPI_connectid(); + execState->m_spiConnectId[execState->m_spiBlockId] = connectid; + execState->m_savedContexts[execState->m_spiBlockId] = CurrentMemoryContext; + MOT_LOG_DEBUG("SPI connect id: depth=%d, saved=%d, cxt=[%s @%p]", + execState->m_spiBlockId, + connectid, + CurrentMemoryContext->name, + CurrentMemoryContext); + ::BeginInternalSubTransaction(NULL); + SubTransactionId subXid = ::GetCurrentSubTransactionId(); + MOT_LOG_DEBUG("Started sub-transaction #%u: id %" PRId64, execState->m_openSubTxCount, subXid); + ++execState->m_openSubTxCount; + execState->m_subTxns[execState->m_spiBlockId] = subXid; + ++execState->m_spiBlockId; + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + (void)HandlePGError(jitContext->m_execState, "SPI_connectid()"); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + if (jitContext->m_execState->m_sqlState == 0) { + // note: we could use a more informative error code + jitContext->m_execState->m_sqlState = ERRCODE_FDW_ERROR; + } + SignalException((JitExec::MotJitContext*)jitContext); + } +} + +void JitEndBlockWithExceptions() +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + + MOT_LOG_DEBUG("Exiting statement block with exceptions"); + if (execState->m_spiBlockId == 0) { + MOT_LOG_ERROR("Cannot end block with exceptions: no block is active"); + RaiseAccessViolationFault(); + } + + PG_TRY(); + { + --execState->m_spiBlockId; + SubTransactionId subXid = ::GetCurrentSubTransactionId(); + if (subXid == execState->m_subTxns[execState->m_spiBlockId]) { + MOT_LOG_DEBUG("Committing current sub-transaction #%" PRIu64 ": id %" PRIu64, + execState->m_openSubTxCount, + (uint64_t)subXid); + // if current txn is already in aborted unrecoverable state then report error + if (u_sess->mot_cxt.jit_txn->IsTxnAborted()) { + raiseAbortTxnError(); + } else { + ::ReleaseCurrentSubTransaction(); + --execState->m_openSubTxCount; + } + } + int savedConnectId = execState->m_spiConnectId[execState->m_spiBlockId]; + MOT_LOG_DEBUG("SPI connect id before calling restore: depth=%d, saved=%d, actual=%d", + execState->m_spiBlockId, + savedConnectId, + SPI_connectid()); + SPI_restore_connection(); + MOT_LOG_DEBUG("SPI connect id after calling restore: %d", SPI_connectid()); + execState->m_spiConnectId[execState->m_spiBlockId] = -1; + MemoryContext savedCxt = execState->m_savedContexts[execState->m_spiBlockId]; + MOT_ASSERT(savedCxt != nullptr); + if (savedCxt == nullptr) { + MOT_LOG_ERROR("Cannot end block with exceptions: saved memory context is null"); + RaiseAccessViolationFault(); + } + (void)MemoryContextSwitchTo(savedCxt); + MOT_LOG_DEBUG("Restored saved memory context: %s @%p", savedCxt->name, savedCxt); + execState->m_savedContexts[execState->m_spiBlockId] = nullptr; + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + (void)HandlePGError(jitContext->m_execState, "SPI_restore_connection()"); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + if (jitContext->m_execState->m_sqlState == 0) { + // note: we could use a more informative error code + jitContext->m_execState->m_sqlState = ERRCODE_FDW_ERROR; + } + SignalException((JitExec::MotJitContext*)jitContext); + } +} + +void JitCleanupBlockAfterException() +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + + MOT_LOG_DEBUG("Cleaning up statement block with exception"); + if (execState->m_spiBlockId == 0) { + MOT_LOG_ERROR("Cannot cleanup block with exceptions: no block is active"); + RaiseAccessViolationFault(); + } + + PG_TRY(); + { + --execState->m_spiBlockId; + SubTransactionId subXid = ::GetCurrentSubTransactionId(); + if (subXid == execState->m_subTxns[execState->m_spiBlockId]) { + MOT_LOG_DEBUG("Rolling back current sub-transaction #%" PRIu64 ": id %" PRIu64, + execState->m_openSubTxCount, + (uint64_t)subXid); + ::RollbackAndReleaseCurrentSubTransaction(); + --execState->m_openSubTxCount; + } + int savedConnectId = execState->m_spiConnectId[execState->m_spiBlockId]; + MOT_LOG_DEBUG("SPI connect id before calling disconnect/restore: depth=%d, saved=%d, actual=%d", + execState->m_spiBlockId, + savedConnectId, + SPI_connectid()); + SPI_disconnect(savedConnectId + 1); + MOT_LOG_DEBUG("SPI connect id after calling disconnect: %d", SPI_connectid()); + SPI_restore_connection(); + MOT_LOG_DEBUG("SPI connect id after calling restore: %d", SPI_connectid()); + execState->m_spiConnectId[execState->m_spiBlockId] = -1; + MemoryContext savedCxt = execState->m_savedContexts[execState->m_spiBlockId]; + MOT_ASSERT(savedCxt != nullptr); + if (savedCxt == nullptr) { + MOT_LOG_ERROR("Cannot cleanup block with exceptions: saved memory context is null"); + RaiseAccessViolationFault(); + } + (void)MemoryContextSwitchTo(savedCxt); + MOT_LOG_DEBUG("Restored saved memory context: %s @%p", savedCxt->name, savedCxt); + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + (void)HandlePGError(jitContext->m_execState, "SPI_restore_connection()"); + signalException = true; + } + PG_END_TRY(); + + if (signalException) { + if (jitContext->m_execState->m_sqlState == 0) { + // note: we could use a more informative error code + jitContext->m_execState->m_sqlState = ERRCODE_FDW_ERROR; + } + SignalException((JitExec::MotJitContext*)jitContext); + } +} + +void JitSetExceptionOrigin(int origin) +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + execState->m_exceptionOrigin = origin; +} + +int JitGetExceptionOrigin() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + return execState->m_exceptionOrigin; +} + +int* JitGetExceptionOriginRef() +{ + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + return &execState->m_exceptionOrigin; +} + +void JitCleanupBeforeReturn(SubTransactionId initSubTxnId) +{ + // close all open blocks (commit sub-txns and release SPI resources) + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + while (execState->m_spiBlockId > 0) { + JitEndBlockWithExceptions(); + if (LlvmGetExceptionStatus() != 0) { + MOT_LOG_DEBUG("JitCleanupBeforeReturn(): Exception thrown, aborting"); + return; + } + } + + // roll back all open sub-transactions (there shouldn't be by now - this is a safety check) + SubTransactionId subTxnId = ::GetCurrentSubTransactionId(); + JitReleaseAllSubTransactions(true, subTxnId); +} + +static JitExec::JitFunctionExecState* GetFunctionExecStateVerify(int subQueryId) +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + JitExec::JitFunctionExecState* execState = GetFunctionExecState(); + JitExec::JitFunctionContext* functionContext = (JitExec::JitFunctionContext*)jitContext; + if (subQueryId >= (int)functionContext->m_SPSubQueryCount) { + MOT_REPORT_ERROR(MOT_ERROR_INDEX_OUT_OF_RANGE, + "Execute JIT Function", + "Attempt to access out-of-range sub-query %d denied: Current function has only %d sub-queries", + subQueryId, + (int)functionContext->m_SPSubQueryCount); + RaiseAccessViolationFault(); + } + return execState; +} + +static void PrepareInvokeParams(JitExec::JitFunctionContext* functionContext, + JitExec::JitFunctionExecState* functionExecState, JitExec::JitInvokedQueryExecState* invokeExecState, + int subQueryId) +{ + // pass parameters coming from stored procedure parameters, some of which may be directly passed from the query + // that invoked the stored procedure + JitExec::JitQueryContext* invokingContext = (JitExec::JitQueryContext*)functionExecState->m_invokingContext; + JitExec::JitQueryExecState* invokingExecState = (JitExec::JitQueryExecState*)invokingContext->m_execState; + JitExec::JitParamInfo* paramInfo = invokingContext->m_invokeParamInfo; + JitExec::JitCallSite* callSite = &functionContext->m_SPSubQueryList[subQueryId]; + MOT_LOG_DEBUG("Using direct params at %p", invokingExecState->m_directParams); + + // note: stored procedure arguments are NOT necessarily starting from index zero + for (int i = 0; i < callSite->m_callParamCount; ++i) { + JitExec::JitCallParamInfo* callParamInfo = &callSite->m_callParamInfo[i]; + int paramId = callParamInfo->m_invokeArgIndex; + // local variables were already passed by jitted SP, we need to pass only SP parameters into called query + if (callParamInfo->m_paramKind == JitExec::JitCallParamKind::JIT_CALL_PARAM_ARG) { + // get SP parameter index from datum index + MOT_ASSERT(paramId < (int)invokingContext->m_invokeParamCount); + if (paramId >= (int)invokingContext->m_invokeParamCount) { + MOT_REPORT_ERROR(MOT_ERROR_INDEX_OUT_OF_RANGE, + "JIT Execute Stored Procedure", + "Invalid parameter index %d, when invoking sub-query %d (%s) of stored procedure %s", + paramId, + subQueryId, + callSite->m_queryContext->m_queryString, + functionContext->m_queryString); + RaiseAccessViolationFault(); + } + if (paramInfo[paramId].m_mode == JitExec::JitParamMode::JIT_PARAM_DIRECT) { + // pass parameter directly from the query that invoked this SP into the called sub-query + // get invoke query parameter index from SP parameter index + int pos = paramInfo[paramId].m_index; + MOT_LOG_DEBUG("Passing direct parameter %d at caller pos %d", i, pos); + invokeExecState->m_params->params[paramId].value = invokingExecState->m_directParams->params[pos].value; + invokeExecState->m_params->params[paramId].isnull = + invokingExecState->m_directParams->params[pos].isnull; + } else { + // pass parameter from the SP itself (since it was modified from INVOKE query into SP) + MOT_LOG_DEBUG("Passing parameter copy %d", i); + invokeExecState->m_params->params[paramId].value = + functionExecState->m_functionParams->params[paramId].value; + invokeExecState->m_params->params[paramId].isnull = + functionExecState->m_functionParams->params[paramId].isnull; + } + } else { + MOT_LOG_DEBUG("Passing local variable %d (already set)", i); + } + if (!invokeExecState->m_params->params[paramId].isnull) { + DEBUG_PRINT_DATUM("Passing parameter to sub-query", + invokeExecState->m_params->params[paramId].ptype, + invokeExecState->m_params->params[paramId].value, + invokeExecState->m_params->params[paramId].isnull); + } + } +} + +static void VerifySpiResult(int rc, JitExec::JitCallSite* callSite) +{ + switch (rc) { + case SPI_OK_SELECT: + case SPI_OK_SELINTO: + case SPI_OK_UTILITY: + case SPI_OK_REWRITTEN: + AssertEreport(!callSite->m_isModStmt, MOD_MOT, "It should not be mod stmt."); + break; + + case SPI_OK_INSERT: + case SPI_OK_UPDATE: + case SPI_OK_DELETE: + case SPI_OK_INSERT_RETURNING: + case SPI_OK_UPDATE_RETURNING: + case SPI_OK_DELETE_RETURNING: + case SPI_OK_MERGE: + AssertEreport(callSite->m_isModStmt, MOD_MOT, "mod stmt is required."); + break; + + case SPI_ERROR_COPY: + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmodule(MOD_MOT), + errmsg("cannot COPY to/from client in PL/pgSQL"))); + break; + + case SPI_ERROR_TRANSACTION: + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmodule(MOD_MOT), + errmsg("only support commit/rollback transaction statements."), + errhint("Use a BEGIN block with an EXCEPTION clause instead of begin/end transaction."))); + break; + + default: + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmodule(MOD_MOT), + errmsg("SPI_execute_plan_with_paramlist failed executing query \"%s\": %s", + callSite->m_queryString, + SPI_result_code_string(rc)))); + break; + } + + if (callSite->m_isInto) { + if (SPI_tuptable == nullptr) { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmodule(MOD_MOT), + errmsg("INTO used with a command that cannot return data"))); + } + } else { + if (SPI_tuptable != nullptr) { + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmodule(MOD_MOT), + errmsg("query has no destination for result data"), + (rc == SPI_OK_SELECT) + ? errhint("If you want to discard the results of a SELECT, use PERFORM instead.") + : 0)); + } + } +} + +// based on convert_value_to_string() from pl_exec.cpp +static char* ConvertValueToString(Datum value, Oid valtype) +{ + char* result = nullptr; + Oid typoutput; + bool typIsVarlena = false; + + getTypeOutputInfo(valtype, &typoutput, &typIsVarlena); + result = OidOutputFunctionCall(typoutput, value); + + return result; +} + +static Datum CastValue(PLpgSQL_type* typeDesc, Datum value, Oid valueType, bool isNull) +{ + Oid funcId = InvalidOid; + CoercionPathType coercionPath = COERCION_PATH_NONE; + Oid requiredType = typeDesc->typoid; + FmgrInfo* requiredInput = &typeDesc->typinput; + Oid requiredTypeIoParam = typeDesc->typioparam; + int32 requiredTypemod = typeDesc->atttypmod; + + if (valueType != requiredType || requiredTypemod != -1) { + if (!isNull) { + if (valueType == UNKNOWNOID) { + valueType = TEXTOID; + value = DirectFunctionCall1(textin, value); + } + + // get the implicit cast function from valueType to requiredType + coercionPath = find_coercion_pathway(requiredType, valueType, COERCION_ASSIGNMENT, &funcId); + if (funcId != InvalidOid && + !(coercionPath == COERCION_PATH_COERCEVIAIO || coercionPath == COERCION_PATH_ARRAYCOERCE)) { + value = OidFunctionCall1(funcId, value); + value = pl_coerce_type_typmod(value, requiredType, requiredTypemod); + } else { + char* extVal = ConvertValueToString(value, valueType); + value = InputFunctionCall(requiredInput, extVal, requiredTypeIoParam, requiredTypemod); + pfree_ext(extVal); + } + } else { + value = InputFunctionCall(requiredInput, NULL, requiredTypeIoParam, requiredTypemod); + } + } + + return value; +} + +static bool CopyHeapTuple( + JitExec::JitInvokedQueryExecState* invokeExecState, HeapTuple tuple, TupleDesc tupleDesc, bool switchContext = true) +{ + // switch to memory context of caller + MemoryContext oldCtx = switchContext ? JitExec::SwitchToSPICallerContext() : nullptr; + + // code adapted from exec_move_row() in pl_exec.cpp + int tupleDescAttrCount = tupleDesc ? tupleDesc->natts : 0; + int tupleAttrCount = HeapTupleIsValid(tuple) ? HeapTupleHeaderGetNatts(tuple->t_data, tupleDesc) : 0; + int resultIndex = 0; + for (int i = 0; i < tupleDescAttrCount; ++i) { + // skip dropped columns in destination + if (tupleDesc->attrs[i].attisdropped) { + continue; + } + + bool isNull = true; + Datum value = (Datum)0; + if ((i < tupleAttrCount) && !tupleDesc->attrs[i].attisdropped) { + value = SPI_getbinval(tuple, tupleDesc, i + 1, &isNull); + if (SPI_result != 0) { + MOT_LOG_TRACE( + "Failed to get datum value from tuple: %s (%u)", SPI_result_code_string(SPI_result), SPI_result); + return false; + } + } else { + continue; // skip dropped columns in source + } + Oid type = SPI_gettypeid(tupleDesc, i + 1); + + // perform proper type conversion as in exec_assign_value() at pl_exec.cpp + PLpgSQL_type* resultType = &invokeExecState->m_resultTypes[resultIndex]; + Datum resValue = CastValue(resultType, value, type, isNull); + resValue = JitExec::CopyDatum(resValue, resultType->typoid, isNull); + SetSlotValue(invokeExecState->m_resultSlot, resultIndex++, resValue, isNull); + } + + // restore current memory context + if (oldCtx) { + MOT_LOG_TRACE("Switching back to memory context %s @%p", oldCtx->name, oldCtx); + (void)MemoryContextSwitchTo(oldCtx); + } + return true; +} + +static void CopyTupleTableSlot(JitExec::JitInvokedQueryExecState* invokeExecState, TupleTableSlot* tuple) +{ + // we might get a minimal tuple from sort result + if (tuple->tts_mintuple != nullptr) { + HeapTuple htup = heap_tuple_from_minimal_tuple(tuple->tts_mintuple); + if (!HeapTupleIsValid(htup)) { + ereport(ERROR, (errmodule(MOD_MOT), errcode(ERRCODE_PLPGSQL_ERROR), errmsg("Failed to form heap tuple"))); + } + if (!CopyHeapTuple(invokeExecState, htup, tuple->tts_tupleDescriptor, false)) { + ereport(ERROR, (errmodule(MOD_MOT), errcode(ERRCODE_PLPGSQL_ERROR), errmsg("Failed to copy heap tuple"))); + } + return; + } + + // switch to memory context of caller (except for when processing minimal tuple of sort query result) + MemoryContext oldCtx = JitExec::SwitchToSPICallerContext(); + + // copy slot values to memory context of caller + TupleDesc tupDesc = tuple->tts_tupleDescriptor; + int resultIndex = 0; + for (int i = 0; i < tupDesc->natts; ++i) { + // skip dropped columns in destination + if (tupDesc->attrs[i].attisdropped) { + continue; + } + + bool isNull = tuple->tts_isnull[i]; + Datum value = tuple->tts_values[i]; + Oid type = tuple->tts_tupleDescriptor->attrs[i].atttypid; + + // perform proper type conversion as in exec_assign_value() at pl_exec.cpp + PLpgSQL_type* resultType = &invokeExecState->m_resultTypes[resultIndex]; + Datum resValue = CastValue(resultType, value, type, isNull); + resValue = JitExec::CopyDatum(resValue, resultType->typoid, isNull); + SetSlotValue(invokeExecState->m_resultSlot, resultIndex++, resValue, isNull); + } + + // restore current memory context + if (oldCtx) { + MOT_LOG_TRACE("Switching back to memory context %s @%p", oldCtx->name, oldCtx); + (void)MemoryContextSwitchTo(oldCtx); + } +} + +inline MOT::RC HandleSubQueryError(JitExec::JitExecState* execState, bool isJittable, int spiConnectId) +{ + const char* operation = isJittable ? "executing jitted SP sub-query" : "executing non-jitted SP sub-query"; + return HandlePGError(execState, operation, spiConnectId); +} + +static bool RecompilePlanIfNeeded( + JitExec::JitFunctionContext* jitContext, int subQueryId, JitExec::JitInvokedQueryExecState* invokeExecState) +{ + if ((invokeExecState->m_plan == nullptr) || needRecompilePlan(invokeExecState->m_plan)) { + MOT_LOG_TRACE("SPI plan needs recompilation"); + // clean up old plan + if (invokeExecState->m_plan != nullptr) { + MOT_LOG_TRACE("Destroying old plan"); + (void)SPI_freeplan(invokeExecState->m_plan); + invokeExecState->m_plan = nullptr; + } + + // regenerate new plan + MOT_LOG_TRACE("Regenerating new plan"); + invokeExecState->m_plan = JitExec::PrepareSpiPlan(jitContext, subQueryId, &invokeExecState->m_expr); + if (invokeExecState->m_plan == nullptr) { + MOT_LOG_TRACE("Failed to prepare SPI plan for non-jittable sub-query %u", subQueryId); + return false; + } + + // do what PG does... + if (ENABLE_CN_GPC && g_instance.plan_cache->CheckRecreateSPICachePlan(invokeExecState->m_plan)) { + g_instance.plan_cache->RecreateSPICachePlan(invokeExecState->m_plan); + } + } + + // setup parameter list for correct parsing (in case we hit parsing in RevalidateCachedQuery) + JitExec::JitFunctionExecState* functionExecState = GetFunctionExecStateVerify(subQueryId); + invokeExecState->m_params->parserSetupArg = (void*)invokeExecState->m_expr; + functionExecState->m_estate.cur_expr = invokeExecState->m_expr; // required by plpgsql_param_fetch + if (invokeExecState->m_expr != nullptr) { + invokeExecState->m_expr->func = functionExecState->m_function; + invokeExecState->m_expr->func->cur_estate = &functionExecState->m_estate; + } + + return true; +} + +static void PrintResultSlot(TupleTableSlot* resultSlot) +{ + if (TTS_EMPTY(resultSlot)) { + MOT_LOG_DEBUG("Query result is empty"); + } else { + MOT_LOG_BEGIN(MOT::LogLevel::LL_DEBUG, "Query result slot:"); + for (int i = 0; i < resultSlot->tts_tupleDescriptor->natts; ++i) { + MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "Datum[%d]: ", i); + Oid type = resultSlot->tts_tupleDescriptor->attrs[i].atttypid; + JitExec::PrintDatum(MOT::LogLevel::LL_DEBUG, type, resultSlot->tts_values[i], resultSlot->tts_isnull[i]); + MOT_LOG_APPEND(MOT::LogLevel::LL_DEBUG, "\n"); + } + MOT_LOG_END(MOT::LogLevel::LL_DEBUG); + } +} + +static inline const char* GetDatumCString(Datum value) +{ + bytea* txt = DatumGetByteaP(value); + return VARDATA(txt); +} + +static inline void ReportErrorFromExecState(JitExec::JitExecState* execState) +{ + void* detailDatum = DatumGetPointer(execState->m_errorDetail); + void* hintDatum = DatumGetPointer(execState->m_errorHint); + if ((detailDatum != nullptr) && (hintDatum != nullptr)) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(execState->m_sqlState), + errmsg("%s", GetDatumCString(execState->m_errorMessage)), + errdetail("%s", GetDatumCString(execState->m_errorDetail)), + errhint("%s", GetDatumCString(execState->m_errorHint)))); + } else if (detailDatum != nullptr) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(execState->m_sqlState), + errmsg("%s", GetDatumCString(execState->m_errorMessage)), + errdetail("%s", GetDatumCString(execState->m_errorDetail)))); + } else if (hintDatum != nullptr) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(execState->m_sqlState), + errmsg("%s", GetDatumCString(execState->m_errorMessage)), + errhint("%s", GetDatumCString(execState->m_errorHint)))); + } else { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(execState->m_sqlState), + errmsg("%s", GetDatumCString(execState->m_errorMessage)))); + } +} + +static int ExecNonJitSubQueryPlan( + JitExec::JitCallSite* callSite, JitExec::JitInvokedQueryExecState* invokeExecState, int tcount, uint32_t subQueryId) +{ + // NOTE: we need to check here for a special use case: this is a jittable query that is executed once as non- + // jittable, and during this execution it was revalidated and executed as jittable through Executor path. + // In this case we need to check whether an error occurred, and if so report it properly + + // in order to make sure order of events described below is correct, we need to make sure error state is reset + if (callSite->m_queryContext != nullptr) { + JitExec::JitExecState* execState = callSite->m_queryContext->m_execState; + if (execState != nullptr) { + ResetErrorState(execState); + } + } + + // execute the query as non-jittable through SPI + // as explained above, it might be revalidated and transform into a jittable query, and get executed as jittable + // through the Executor path + int rc = SPI_execute_plan_with_paramlist(invokeExecState->m_plan, invokeExecState->m_params, false, tcount); + + // first check if this was a non-jittable execution of an invalid jittable query + if (callSite->m_queryContext != nullptr) { + // we avoid checking the validity of the query (so we have indication the jittable execution took place), since + // the valid bit might be overwritten (context invalidated) after jitted execution. + // instead, we just check the SQL state in the execution state - if it is not zero then definitely a jitted + // execution took place and it failed (the use case of non-failed jitted execution is of no interest here) + JitExec::JitExecState* execState = callSite->m_queryContext->m_execState; + if ((execState != nullptr) && (execState->m_sqlState != 0)) { // the execution ended with error + // query was executed as jittable and an error occurred, so we throw the same error again + ReportErrorFromExecState(execState); + } + } + + // next check return code from SPI execution + invokeExecState->m_spiResult = rc; + VerifySpiResult(rc, callSite); + rc = 0; // we are good, report success for now + + // now handle result tuple if needed + invokeExecState->m_tuplesProcessed = SPI_processed; + MOT_LOG_TRACE("Sub-query %u returned %u tuples processed: %s", subQueryId, SPI_processed, callSite->m_queryString); + if ((callSite->m_queryCmdType == CMD_SELECT) && (SPI_processed > 0)) { + SPITupleTable* tupTab = SPI_tuptable; + if (!CopyHeapTuple(invokeExecState, tupTab->vals[0], tupTab->tupdesc)) { + rc = SPI_ERROR_OPUNKNOWN; + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_PLPGSQL_ERROR), + errmsg("Failed to copy result tuple to caller's context for non-jittable sub-query %u: %s", + subQueryId, + callSite->m_queryString))); + } + if (MOT_CHECK_DEBUG_LOG_LEVEL()) { + PrintResultSlot(invokeExecState->m_resultSlot); + } + } + + // cleanup + if (SPI_tuptable != nullptr) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = nullptr; + } + return rc; +} + +static int JitExecNonJitSubQuery(int subQueryId, int tcount) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile int result = 0; + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile bool isSpiPushed = false; + volatile int spiConnectId = SPI_connectid(); + + JitExec::JitFunctionExecState* functionExecState = GetFunctionExecStateVerify(subQueryId); + JitExec::JitFunctionContext* functionContext = (JitExec::JitFunctionContext*)jitContext; + JitExec::JitCallSite* callSite = &functionContext->m_SPSubQueryList[subQueryId]; + JitExec::JitInvokedQueryExecState* invokeExecState = &functionExecState->m_invokedQueryExecState[subQueryId]; + bool planSourceHadContext = false; + if (invokeExecState->m_plan != nullptr && invokeExecState->m_plan->plancache_list != NIL) { + CachedPlanSource* planSource = (CachedPlanSource*)linitial(invokeExecState->m_plan->plancache_list); + if (planSource->mot_jit_context != nullptr) { + planSourceHadContext = true; + } + } + + // get a valid query string + const char* queryString = callSite->m_queryString; + if ((queryString == nullptr) && (callSite->m_queryContext != nullptr)) { + queryString = callSite->m_queryContext->m_queryString; + } + MOT_ASSERT(queryString != nullptr); + + MOT_LOG_DEBUG("Executing non-jittable sub-query: %s", queryString) + jitContext->m_execState->m_sqlState = 0; + PG_TRY(); + { + // run sub-query to completion, in strict execution it is expected to return only one tuple, otherwise none. + MOT_LOG_DEBUG("Invoking SP non-jittable sub-query %d with params: %p", subQueryId, invokeExecState->m_params); + + // we push SPI context only if we expect to call another SP, + // otherwise we continue working in current memory context + if (callSite->m_isUnjittableInvoke) { + MOT_LOG_DEBUG("Calling SPI_push()") + SPI_push(); + isSpiPushed = true; + } + + // ensure we have a prepared SPI plan + if (!RecompilePlanIfNeeded(functionContext, subQueryId, invokeExecState)) { + MOT_LOG_TRACE("Failed to recompile SPI plan for non-jittable sub-query %u: %s", subQueryId, queryString); + // throw ereport to get proper error handling + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_PLPGSQL_ERROR), + errmsg("Failed to recompile SPI plan for non-jittable sub-query %u: %s", + subQueryId, + callSite->m_queryString))); + } else if (invokeExecState->m_plan == nullptr) { + MOT_LOG_TRACE("Found unprepared SPI plan for non-jittable sub-query %u: %s", subQueryId, queryString); + // throw ereport to get proper error handling + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_PLPGSQL_ERROR), + errmsg("Found unprepared SPI plan for non-jittable sub-query %u: %s", subQueryId, queryString))); + } else { + // prepare invoke parameters + PrepareInvokeParams(functionContext, functionExecState, invokeExecState, subQueryId); + + // now execute the plan + (void)::ExecClearTuple(invokeExecState->m_resultSlot); + result = ExecNonJitSubQueryPlan(callSite, invokeExecState, tcount, subQueryId); + if (result != 0) { + // throw ereport to get proper error handling + MOT_LOG_TRACE("Failed to execute non-jittable sub-query %u: %s", subQueryId, queryString); + if (jitContext->m_execState->m_sqlState == 0) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_PLPGSQL_ERROR), + errmsg("Failed to execute non-jittable sub-query %u: %s", subQueryId, queryString))); + } else { + signalException = true; + } + } + MOT_LOG_DEBUG("Finished executing non-jittable SP sub-query: %s", queryString); + } + + // ATTENTION: it is possible that we arrived here through a jittable sub-query in intermediate state + // (e.g. pending compile), and due to revalidation the jit context was deleted in the plan, so we must update + // the call site, otherwise we will crash during context destruction + CachedPlanSource* planSource = (CachedPlanSource*)linitial(invokeExecState->m_plan->plancache_list); + if ((planSourceHadContext) && (planSource->mot_jit_context == nullptr) && + (callSite->m_queryContext != nullptr)) { + // transform into non-jittable sub-query + // the query string is safe to duplicate, as it comes from the JIT source + callSite->m_queryContext = nullptr; + if (callSite->m_queryString == nullptr) { + callSite->m_queryString = JitExec::DupString(queryString, JitExec::JitContextUsage::JIT_CONTEXT_LOCAL); + if (callSite->m_queryString == nullptr) { + ereport(ERROR, + (errmodule(MOD_MOT), + errmsg("Failed to allocate local memory for query string: %s", queryString))); + } + } + } + } + PG_CATCH(); + { + (void)MemoryContextSwitchTo(origCxt); + HandleSubQueryError(jitContext->m_execState, false, spiConnectId); + isSpiPushed = false; // SPI connection already restored to original state in HandlePGError() + signalException = true; + } + PG_END_TRY(); + + u_sess->mot_cxt.jit_txn->FinishStatement(); + + if (isSpiPushed) { + SPI_pop(); + } + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + + // ATTENTION: original SPI result is passed to caller for further inspection through the execution state of the + // backing context + return result; +} + +void JitReleaseNonJitSubQueryResources(int subQueryId) +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + JitExec::JitFunctionContext* functionContext = (JitExec::JitFunctionContext*)jitContext; + JitExec::JitCallSite* callSite = &functionContext->m_SPSubQueryList[subQueryId]; + JitExec::JitQueryContext* subQueryContext = (JitExec::JitQueryContext*)callSite->m_queryContext; + if (subQueryContext == nullptr) { + if (SPI_tuptable != nullptr) { + SPI_freetuptable(SPI_tuptable); + SPI_tuptable = nullptr; + } + } +} + +int JitExecSubQuery(int subQueryId, int tcount) +{ + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile JitExec::JitFunctionExecState* functionExecState = GetFunctionExecStateVerify(subQueryId); + JitExec::JitFunctionContext* functionContext = (JitExec::JitFunctionContext*)jitContext; + JitExec::JitCallSite* callSite = &functionContext->m_SPSubQueryList[subQueryId]; + volatile JitExec::JitQueryContext* subQueryContext = (JitExec::JitQueryContext*)callSite->m_queryContext; + + MOT_LOG_DEBUG("Executing sub-query %d with tcount %d", subQueryId, tcount); + if ((subQueryContext == nullptr) || IsJitContextPendingCompile((JitExec::MotJitContext*)subQueryContext) || + IsJitContextErrorCompile((JitExec::MotJitContext*)subQueryContext)) { + MOT_LOG_DEBUG("Executing non-jittable sub-query %u: %s", subQueryId, callSite->m_queryString); + return JitExecNonJitSubQuery(subQueryId, tcount); + } + + // if context is invalid it will be revalidated when we acquire locks, and if that fails we will transform into + // non-jittable sub-query + MOT_LOG_DEBUG("Executing jittable sub-query %u: %s", subQueryId, subQueryContext->m_queryString); + + // ATTENTION: current sub-query id must be exposed so that result collection can take place + functionExecState->m_currentSubQueryId = subQueryId; + + // function parameters already passed into sub-query parameters by previous instructions + + // ATTENTION: sub-query result slot is maintained in sub-query exec state + + // ATTENTION: current call might generate ereport that will end entire function execution, so we need to catch it + // and translate it so it an be caught by exception handler block in jitted code + jitContext->m_execState->m_sqlState = 0; + subQueryContext->m_execState->m_sqlState = 0; + MOT_LOG_DEBUG("(Before sub-query) Current sub-transaction #%" PRIu64 ": id %" PRIu64, + functionExecState->m_openSubTxCount, + (uint64_t)::GetCurrentSubTransactionId()); + + // in case we hit parsing during revalidate, we must make sure we have up-to-date parameters + volatile JitExec::JitInvokedQueryExecState* invokeExecState = + &functionExecState->m_invokedQueryExecState[subQueryId]; + invokeExecState->m_params->parserSetupArg = (void*)invokeExecState->m_expr; + functionExecState->m_estate.cur_expr = invokeExecState->m_expr; // required by plpgsql_param_fetch + if (invokeExecState->m_expr != nullptr) { + invokeExecState->m_expr->func = functionExecState->m_function; + invokeExecState->m_expr->func->cur_estate = (PLpgSQL_execstate*)&functionExecState->m_estate; + } + MOT_LOG_DEBUG("Invoking SP sub-query %d with params: %p", subQueryId, invokeExecState->m_params); + invokeExecState->m_spiResult = SPI_OK_SELECT; + + // every variable used after catch needs to be volatile (see longjmp() man page) + volatile int result = 0; + volatile bool signalException = false; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile SubTransactionId subXid = ::GetCurrentSubTransactionId(); + volatile CachedPlan* cplan = nullptr; + volatile bool planTransformed = false; + volatile bool planExecuteAsNonJit = false; + volatile bool pushedActiveSnapshot = false; + volatile bool isSpiPushed = false; + volatile int spiConnectId = SPI_connectid(); + + // run sub-query to completion, in strict execution it is expected to return only one tuple, otherwise none. + PG_TRY(); + { + // acquire plan locks before execution - this may trigger JIT context revalidation (this is good) + // attention: we do not need full analysis, just revalidation and parse + if (subQueryContext->m_commandType != JitExec::JIT_COMMAND_INVOKE) { + // we need to make sure the plan source points to the jit context to be executed, so revalidation occurs + const char* queryString = subQueryContext->m_jitSource->m_queryString; + MOT_ASSERT(list_length(invokeExecState->m_plan->plancache_list) == 1); + CachedPlanSource* planSource = (CachedPlanSource*)linitial(invokeExecState->m_plan->plancache_list); + MOT_ASSERT(planSource->mot_jit_context == nullptr || (planSource->mot_jit_context == subQueryContext)); + if (planSource->mot_jit_context == nullptr) { + planSource->mot_jit_context = (JitExec::MotJitContext*)subQueryContext; + } + + cplan = SPI_plan_get_cached_plan(invokeExecState->m_plan); + if (cplan == nullptr) { + MOT_LOG_ERROR("Failed to get cached plan"); + } + + // it is possible at this point that query got disqualified and is not jittable now + if (planSource->mot_jit_context == nullptr) { + // we transform this sub-query into a non-jittable one and then execute it + // the query string is safe to duplicate, as it comes from the JIT source + MOT_LOG_TRACE("Jitted sub-query became unjittable after revalidation: %s", queryString); + callSite->m_queryString = JitExec::DupString(queryString, JitExec::JitContextUsage::JIT_CONTEXT_LOCAL); + if (callSite->m_queryString == nullptr) { + ereport(ERROR, + (errmodule(MOD_MOT), + errmsg("Failed to allocate local memory for query string: %s", queryString))); + } + callSite->m_queryContext = nullptr; // already deleted during revalidation + planTransformed = true; + } else if (!JitExec::IsJitContextValid(planSource->mot_jit_context)) { + MOT_LOG_TRACE("Jitted sub-query revalidation failed, executing as non-jittable: %s", queryString); + planExecuteAsNonJit = true; + } + } else { + if (IsJitContextDoneCompile((JitExec::MotJitContext*)subQueryContext)) { + MOT_LOG_TRACE("Revalidate sub-query %d after compile-done: %s", + subQueryId, + subQueryContext->m_jitSource->m_queryString); + if (!RevalidateJitContext((JitExec::MotJitContext*)subQueryContext)) { + MOT_LOG_TRACE("Revalidate sub-query %d after compile-done failed, executing as non-jittable: %s", + subQueryId, + subQueryContext->m_jitSource->m_queryString); + planExecuteAsNonJit = true; + } + } + } + + if (planTransformed || planExecuteAsNonJit) { + if (planTransformed) { + MOT_ASSERT(callSite->m_queryString != nullptr); + MOT_LOG_TRACE("Executing transformed SP query as non-jittable: %s", callSite->m_queryString); + } else { + MOT_LOG_TRACE( + "Executing invalid SP query as non-jittable: %s", callSite->m_queryContext->m_queryString); + } + result = JitExecNonJitSubQuery(subQueryId, tcount); + } else { + PushActiveSnapshot(GetTransactionSnapshot()); + pushedActiveSnapshot = false; + + // we push SPI context only if we expect to call another SP, + // otherwise we continue working in current memory context + if (subQueryContext->m_commandType == JitExec::JIT_COMMAND_INVOKE) { + SPI_push(); + isSpiPushed = true; + } + + // run sub-query to completion, in strict execution it is expected to return only one tuple, otherwise none. + JitExec::JitInvokedQueryExecState* subQueryExecState = + &functionExecState->m_invokedQueryExecState[subQueryId]; + MOT_LOG_DEBUG("Invoking SP sub-query %d with params: %p", subQueryId, subQueryExecState->m_params); + + // pass parameters coming from stored procedure parameters, some of which may be directly passed from the + // query that invoked the stored procedure + PrepareInvokeParams( + functionContext, (JitExec::JitFunctionExecState*)functionExecState, subQueryExecState, subQueryId); + + // setup invoking context + subQueryContext->m_execState->m_invokingContext = functionContext; + + // make sure result is cleared - so caller can tell zero tuples were retrieved + (void)::ExecClearTuple(subQueryExecState->m_resultSlot); + JitResetScan((JitExec::MotJitContext*)subQueryContext); // signal new scan is starting + if ((subQueryContext->m_commandType == JitExec::JIT_COMMAND_RANGE_SELECT) || + (subQueryContext->m_commandType == JitExec::JIT_COMMAND_RANGE_JOIN)) { + unsigned long nprocessed = 0; + // save only first slot for strict execution, the rest can be overwritten on work slot + TupleTableSlot* slot = subQueryExecState->m_resultSlot; + bool finish = false; + while (!finish) { + uint64_t tuplesProcessed = 0; + result = JitExec::JitExecQuery((JitExec::MotJitContext*)subQueryContext, + subQueryExecState->m_params, + slot, + &tuplesProcessed, + &subQueryExecState->m_scanEnded); + if (subQueryExecState->m_scanEnded || (tuplesProcessed == 0) || (result != 0)) { + // raise flag so that next round we will bail out (current tuple still must be reported to user) + finish = true; + } + if (tuplesProcessed > 0) { + ++nprocessed; + if ((tcount != 0) && (nprocessed == (unsigned long)tcount)) { + finish = true; + } + } + // copy first slot if needed and switch to work slot from now on + if (slot == subQueryExecState->m_resultSlot) { + // we need to copy slot if there is a minimal tuple or SPI push was called and the tuple is not + // empty + if ((slot->tts_mintuple != nullptr) || (isSpiPushed && !TTS_EMPTY(slot))) { + CopyTupleTableSlot(subQueryExecState, subQueryExecState->m_resultSlot); + } + } + slot = subQueryExecState->m_workSlot; + } + subQueryExecState->m_tuplesProcessed = nprocessed; + if ((nprocessed > 0) && (result == MOT::RC_LOCAL_ROW_NOT_FOUND)) { + // if the first rounds were fine and retrieved rows, but the last iteration found no row, then we + // should not report any error - this is normal + result = MOT::RC_OK; + } + } else { + // in this case we disregard tcount, since this is a point query, or even insert/delete/update + result = JitExec::JitExecQuery((JitExec::MotJitContext*)subQueryContext, + subQueryExecState->m_params, + subQueryExecState->m_resultSlot, + &subQueryExecState->m_tuplesProcessed, + &subQueryExecState->m_scanEnded); + } + + CommandCounterIncrement(); + PopActiveSnapshot(); + pushedActiveSnapshot = false; + + MOT_LOG_DEBUG("Finished executing jitted SP sub-query with %" PRIu64 " tuples and result: %s", + subQueryExecState->m_tuplesProcessed, + MOT::RcToString((MOT::RC)result)); + if ((result != MOT::RC_OK) && (result != MOT::RC_LOCAL_ROW_NOT_FOUND)) { + // throw ereport to get proper error handling + MOT_LOG_TRACE( + "Failed to execute jittable sub-query %u: %s", subQueryId, subQueryContext->m_queryString); + if (subQueryContext->m_execState->m_sqlState == 0) { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_PLPGSQL_ERROR), + errmsg("Failed to execute jittable sub-query %u: %s", + subQueryId, + subQueryContext->m_queryString))); + } + } else if (result != MOT::RC_LOCAL_ROW_NOT_FOUND) { + if (MOT_CHECK_DEBUG_LOG_LEVEL()) { + PrintResultSlot(subQueryExecState->m_resultSlot); + } + } + // cleanup SPI stack if needed before handling any error + if (isSpiPushed) { + SPI_pop(); + isSpiPushed = false; + } + } + + // release plan - locks are released by resource owner during commit! + if (cplan != nullptr) { + ReleaseCachedPlan((CachedPlan*)cplan, invokeExecState->m_plan->saved); + } + } + PG_CATCH(); + { + // first restore memory context + CurrentMemoryContext = origCxt; + + // release cached plan or planner locks + if (cplan != nullptr) { + ReleaseCachedPlan((CachedPlan*)cplan, invokeExecState->m_plan->saved); + } + + // print error + HandleSubQueryError(subQueryContext->m_execState, true, spiConnectId); + isSpiPushed = false; // SPI connection already restored to original state in HandlePGError() + signalException = true; + + if (pushedActiveSnapshot) { + PopActiveSnapshot(); + } + + // in debug mode we validate sub-tx state + SubTransactionId endSubXid = ::GetCurrentSubTransactionId(); + if (subXid != endSubXid) { + MOT_LOG_WARN("Invalid sub-tx state: entered with %" PRIu64 ", exited with " PRIu64, subXid, endSubXid); + } + MOT_ASSERT(subXid == endSubXid); + // rollback all open sub-txns until the one we started with + } + PG_END_TRY(); + + MOT_ASSERT(!isSpiPushed); + // pull-up error info + if (!planTransformed && !planExecuteAsNonJit) { + PullUpErrorInfo(subQueryContext->m_execState, jitContext->m_execState); + u_sess->mot_cxt.jit_txn->FinishStatement(); + } + + // reset sub-query id + functionExecState->m_currentSubQueryId = -1; + + if (!planTransformed && !planExecuteAsNonJit && signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + + // return execution result + return result; +} + +int JitGetTuplesProcessed(int subQueryId) +{ + JitExec::JitFunctionExecState* functionExecState = GetFunctionExecStateVerify(subQueryId); + JitExec::JitInvokedQueryExecState* invokeExecState = &functionExecState->m_invokedQueryExecState[subQueryId]; + return invokeExecState->m_tuplesProcessed; +} + +Datum JitGetSubQuerySlotValue(int subQueryId, int tupleColId) +{ + JitExec::JitFunctionExecState* functionExecState = GetFunctionExecStateVerify(subQueryId); + TupleTableSlot* slot = functionExecState->m_invokedQueryExecState[subQueryId].m_resultSlot; + return GetSlotValue(slot, tupleColId); +} + +int JitGetSubQuerySlotIsNull(int subQueryId, int tupleColId) +{ + JitExec::JitFunctionExecState* functionExecState = GetFunctionExecStateVerify(subQueryId); + TupleTableSlot* slot = functionExecState->m_invokedQueryExecState[subQueryId].m_resultSlot; + return GetSlotIsNull(slot, tupleColId); +} + +HeapTuple JitGetSubQueryResultHeapTuple(int subQueryId) +{ + HeapTuple heapTuple = (HeapTuple)DatumGetPointer(JitGetSubQuerySlotValue(subQueryId, 0)); + MOT_LOG_DEBUG("Retrieving heap tuple for sub-query %d at %p", subQueryId, heapTuple); + return heapTuple; +} + +Datum JitGetHeapTupleValue(HeapTuple tuple, int subQueryId, int columnId, int* isNull) +{ + MOT_ASSERT(u_sess->mot_cxt.jit_context->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + JitExec::JitFunctionContext* jitContext = (JitExec::JitFunctionContext*)u_sess->mot_cxt.jit_context; + MOT_ASSERT(subQueryId < (int)jitContext->m_SPSubQueryCount); + JitExec::MotJitContext* subContext = jitContext->m_SPSubQueryList[subQueryId].m_queryContext; + MOT_ASSERT(subContext->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_QUERY); + MOT_ASSERT(subContext->m_commandType == JitExec::JIT_COMMAND_INVOKE); + TupleDesc tupDesc = ((JitExec::JitQueryContext*)subContext)->m_invokeContext->m_resultTupDesc; + + MOT_LOG_DEBUG("Retrieving sub-query %d heap tuple %p attribute %d using tuple descriptor %p", + subQueryId, + tuple, + columnId, + tupDesc); + bool isNullBool = false; + Datum value = SPI_getbinval(tuple, tupDesc, columnId + 1, &isNullBool); + *isNull = isNullBool ? 1 : 0; + DEBUG_PRINT_DATUM("Datum: ", tupDesc->attrs[columnId]->atttypid, value, isNullBool); + return value; +} + +int JitGetSpiResult(int subQueryId) +{ + JitExec::JitFunctionExecState* functionExecState = GetFunctionExecStateVerify(subQueryId); + JitExec::JitInvokedQueryExecState* invokeExecState = &functionExecState->m_invokedQueryExecState[subQueryId]; + int rc = invokeExecState->m_spiResult; + MOT_LOG_DEBUG("Returning sub-query SPI result %d: %s", rc, SPI_result_code_string(rc)); + return rc; +} + +inline Datum JitConvertViaStringImpl(Datum value, Oid resultType, Oid targetType, int typeMod) +{ + // convert result type to target type via string + // convert result to string using type output function + Oid typeOutput; + bool typeIsVarlena = false; + getTypeOutputInfo(resultType, &typeOutput, &typeIsVarlena); + char* valueStr = OidOutputFunctionCall(typeOutput, value); + MOT_LOG_DEBUG("JitConvertViaString(): Intermediate str result = %s", valueStr); + + // convert string to target type using type input function + Oid typeInput; + Oid typeIoParam; + getTypeInputInfo(targetType, &typeInput, &typeIoParam); + Datum result = OidInputFunctionCall(typeInput, valueStr, typeIoParam, typeMod); + pfree_ext(valueStr); + + return result; +} + +Datum JitConvertViaString(Datum value, Oid resultType, Oid targetType, int typeMod) +{ + MOT_LOG_DEBUG("JitConvertViaString() resultType = %d, targetType=%d, typeMod=%d", resultType, targetType, typeMod); + + volatile Datum result = PointerGetDatum(nullptr); + volatile bool signalException = false; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + volatile JitExec::JitFunctionExecState* execState = (JitExec::JitFunctionExecState*)jitContext->m_execState; + volatile MemoryContext origCxt = (execState->m_spiBlockId >= 1) + ? MemoryContextSwitchTo(execState->m_savedContexts[0]) + : JitExec::SwitchToSPICallerContext(); + volatile int spiConnectId = SPI_connectid(); + PG_TRY(); + { + result = JitConvertViaStringImpl(value, resultType, targetType, typeMod); + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + if (origCxt != nullptr) { + (void)MemoryContextSwitchTo(origCxt); + } + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + return result; +} + +static Datum JitCastValueImpl(Datum value, Oid sourceType, Oid targetType, int typeMod, CoercionPathType coercePath, + Oid funcId, CoercionPathType coercePath2, Oid funcId2, int nargs, bool& exceptionSignaled) +{ + // temporary calculation made on current memory context + volatile Datum result = PointerGetDatum(nullptr); + volatile bool signalException = false; + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + volatile MemoryContext origCxt = CurrentMemoryContext; + volatile int spiConnectId = SPI_connectid(); + PG_TRY(); + { + MOT_ASSERT((targetType != sourceType) || (typeMod != -1)); + if ((funcId != InvalidOid) && + !(coercePath == COERCION_PATH_COERCEVIAIO || coercePath == COERCION_PATH_ARRAYCOERCE)) { + // call convert once + result = OidFunctionCall1(funcId, value); + if ((coercePath2 == COERCION_PATH_FUNC) && OidIsValid(funcId2)) { + if (nargs == 1) { + result = OidFunctionCall1(funcId2, result); + } else if (nargs == 2) { + result = OidFunctionCall2(funcId2, result, typeMod); + } else if (nargs == 3) { + result = OidFunctionCall3(funcId2, result, typeMod, 0); + } else { + ereport(ERROR, + (errmodule(MOD_MOT), + errcode(ERRCODE_OUT_OF_LOGICAL_MEMORY), + errmsg("Unexpected number of arguments for conversion function: %u", funcId))); + } + } + } else { + result = JitConvertViaStringImpl(value, sourceType, targetType, typeMod); + } + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + if (origCxt != nullptr) { + (void)MemoryContextSwitchTo(origCxt); + } + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + exceptionSignaled = true; + } + + return result; +} + +Datum JitCastValue(Datum value, Oid sourceType, Oid targetType, int typeMod, int coercePath, Oid funcId, + int coercePath2, Oid funcId2, int nargs, int typeByVal) +{ + volatile Datum result = PointerGetDatum(nullptr); + bool exceptionSignaled = false; + + MOT_LOG_DEBUG("Casting value from type %u to type %u by typeMod %d, typeByVal = %d", + sourceType, + targetType, + typeMod, + typeByVal); + + // make the cast + if ((targetType != sourceType) || (typeMod != -1)) { + result = JitCastValueImpl(value, + sourceType, + targetType, + typeMod, + (CoercionPathType)coercePath, + funcId, + (CoercionPathType)coercePath2, + funcId2, + nargs, + exceptionSignaled); + + // check if exception occurred + if (exceptionSignaled) { + return result; + } + } else { + // direct assignment - no cast needed (impossible when typeByVal == 0, because in this case we would not emit + // call to JitCastValue in the first place...) + MOT_ASSERT(!typeByVal); + result = value; + } + + // if type is passed by value we are done + if (typeByVal) { + return result; + } + + // copy datum to caller's memory context + volatile JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext->m_contextType == JitExec::JitContextType::JIT_CONTEXT_TYPE_FUNCTION); + volatile JitExec::JitFunctionExecState* execState = (JitExec::JitFunctionExecState*)jitContext->m_execState; + volatile MemoryContext origCxt = (execState->m_spiBlockId >= 1) + ? MemoryContextSwitchTo(execState->m_savedContexts[0]) + : JitExec::SwitchToSPICallerContext(); + volatile int spiConnectId = SPI_connectid(); + volatile bool signalException = false; + PG_TRY(); + { + result = JitExec::CopyDatum(result, targetType, false); + } + PG_CATCH(); + { + // switch back to original context before issuing an error report + (void)MemoryContextSwitchTo(origCxt); + HandlePGFunctionError((JitExec::MotJitContext*)jitContext, spiConnectId); + signalException = true; + } + PG_END_TRY(); + if (origCxt != nullptr) { + (void)MemoryContextSwitchTo(origCxt); + } + + if (signalException) { + SignalException((JitExec::MotJitContext*)jitContext); + } + + return result; +} + +void JitSaveErrorInfo(Datum errorMessage, int sqlState, Datum sqlStateString) +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext != nullptr); + JitExec::JitExecState* execState = jitContext->m_execState; + MOT_ASSERT(execState != nullptr); + execState->m_errorMessage = errorMessage; + execState->m_sqlState = sqlState; + execState->m_sqlStateString = sqlStateString; +} + +Datum JitGetErrorMessage() +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext != nullptr); + JitExec::JitExecState* execState = jitContext->m_execState; + MOT_ASSERT(execState != nullptr); + return execState->m_errorMessage; +} + +int JitGetSqlState() +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext != nullptr); + JitExec::JitExecState* execState = jitContext->m_execState; + MOT_ASSERT(execState != nullptr); + return execState->m_sqlState; +} + +Datum JitGetSqlStateString() +{ + JitExec::MotJitContext* jitContext = u_sess->mot_cxt.jit_context; + MOT_ASSERT(jitContext != nullptr); + JitExec::JitExecState* execState = jitContext->m_execState; + MOT_ASSERT(execState != nullptr); + return execState->m_sqlStateString; +} + +Datum JitGetDatumIsNotNull(int isNull) +{ + return BoolGetDatum(!isNull ? true : false); +} + +void EmitProfileData(uint32_t functionId, uint32_t regionId, int beginRegion) +{ + JitExec::JitProfiler::GetInstance()->EmitProfileData(functionId, regionId, beginRegion != 0); } } // extern "C" diff --git a/src/gausskernel/storage/mot/jit_exec/jit_llvm_blocks.cpp b/src/gausskernel/storage/mot/jit_exec/jit_llvm_blocks.cpp index b3e91d387..47ba67c16 100644 --- a/src/gausskernel/storage/mot/jit_exec/jit_llvm_blocks.cpp +++ b/src/gausskernel/storage/mot/jit_exec/jit_llvm_blocks.cpp @@ -13,2270 +13,620 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * jit_llvm_blocks.cpp - * Helpers to generate compound LLVM code. + * occ_transaction_manager.cpp + * Optimistic Concurrency Control (OCC) implementation * * IDENTIFICATION - * src/gausskernel/storage/mot/jit_exec/jit_llvm_blocks.cpp + * src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.h * * ------------------------------------------------------------------------- */ -/* - * ATTENTION: Be sure to include jit_llvm_query.h before anything else because of gscodegen.h - * (jit_llvm_blocks.h includes jit_llvm_query.h before anything else). - * See jit_llvm_query.h for more details. - */ -#include "jit_llvm_blocks.h" -#include "jit_llvm_funcs.h" -#include "jit_util.h" -#include "mot_error.h" +#include "occ_transaction_manager.h" #include "utilities.h" +#include "cycles.h" +#include "mot_engine.h" +#include "row.h" +#include "txn.h" +#include "txn_access.h" +#include "checkpoint_manager.h" +#include "mm_session_api.h" +#include "mot_error.h" +#include -#include "catalog/pg_aggregate.h" +namespace MOT { +DECLARE_LOGGER(OccTransactionManager, ConcurrenyControl); -using namespace dorado; +OccTransactionManager::OccTransactionManager() + : m_txnCounter(0), + m_abortsCounter(0), + m_writeSetSize(0), + m_insertSetSize(0), + m_dynamicSleep(100), + m_rowsLocked(false), + m_preAbort(true), + m_validationNoWait(true), + m_isTransactionCommited(false) +{} -namespace JitExec { -DECLARE_LOGGER(JitLlvmBlocks, JitExec) +OccTransactionManager::~OccTransactionManager() +{} -static bool ProcessJoinOpExpr( - JitLlvmCodeGenContext* ctx, const OpExpr* op_expr, int* column_count, int* column_array, int* max_arg); -static bool ProcessJoinBoolExpr( - JitLlvmCodeGenContext* ctx, const BoolExpr* boolexpr, int* column_count, int* column_array, int* max_arg); -static llvm::Value* ProcessFilterExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitFilter* filter, int* max_arg); -static llvm::Value* ProcessExpr( - JitLlvmCodeGenContext* ctx, Expr* expr, int& result_type, int arg_pos, int depth, int* max_arg); -static llvm::Value* ProcessExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitExpr* expr, int* max_arg); - -/*--------------------------- Helpers to generate compound LLVM code ---------------------------*/ -/** @brief Creates a jitted function for code generation. Builds prototype and entry block. */ -void CreateJittedFunction(JitLlvmCodeGenContext* ctx, const char* function_name) +bool OccTransactionManager::PreAbortCheck(TxnManager* txMan, GcMaintenanceInfo& gcMemoryReserve) { - llvm::Value* llvmargs[MOT_JIT_FUNC_ARG_COUNT]; - - // define the function prototype - GsCodeGen::FnPrototype fn_prototype(ctx->_code_gen, function_name, ctx->INT32_T); - fn_prototype.addArgument(GsCodeGen::NamedVariable("table", ctx->TableType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("index", ctx->IndexType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("key", ctx->KeyType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("bitmap", ctx->BitmapSetType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("params", ctx->ParamListInfoDataType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("slot", ctx->TupleTableSlotType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("tp_processed", ctx->INT64_T->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("scan_ended", ctx->INT32_T->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("isNewScan", ctx->INT32_T)); - fn_prototype.addArgument(GsCodeGen::NamedVariable("end_iterator_key", ctx->KeyType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("inner_table", ctx->TableType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("inner_index", ctx->IndexType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("inner_key", ctx->KeyType->getPointerTo())); - fn_prototype.addArgument(GsCodeGen::NamedVariable("inner_end_iterator_key", ctx->KeyType->getPointerTo())); - - ctx->m_jittedQuery = fn_prototype.generatePrototype(ctx->_builder, &llvmargs[0]); - - // get the arguments - int arg_index = 0; - ctx->table_value = llvmargs[arg_index++]; - ctx->index_value = llvmargs[arg_index++]; - ctx->key_value = llvmargs[arg_index++]; - ctx->bitmap_value = llvmargs[arg_index++]; - ctx->params_value = llvmargs[arg_index++]; - ctx->slot_value = llvmargs[arg_index++]; - ctx->tp_processed_value = llvmargs[arg_index++]; - ctx->scan_ended_value = llvmargs[arg_index++]; - ctx->isNewScanValue = llvmargs[arg_index++]; - ctx->end_iterator_key_value = llvmargs[arg_index++]; - ctx->inner_table_value = llvmargs[arg_index++]; - ctx->inner_index_value = llvmargs[arg_index++]; - ctx->inner_key_value = llvmargs[arg_index++]; - ctx->inner_end_iterator_key_value = llvmargs[arg_index++]; - - for (uint32_t i = 0; i < ctx->m_subQueryCount; ++i) { - ctx->m_subQueryData[i].m_slot = AddGetSubQuerySlot(ctx, i); - ctx->m_subQueryData[i].m_table = AddGetSubQueryTable(ctx, i); - ctx->m_subQueryData[i].m_index = AddGetSubQueryIndex(ctx, i); - ctx->m_subQueryData[i].m_searchKey = AddGetSubQuerySearchKey(ctx, i); - ctx->m_subQueryData[i].m_endIteratorKey = AddGetSubQueryEndIteratorKey(ctx, i); - } - - IssueDebugLog("Starting execution of jitted function"); -} - -/** @brief Builds a code segment for checking if soft memory limit has been reached. */ -void buildIsSoftMemoryLimitReached(JitLlvmCodeGenContext* ctx) -{ - JIT_IF_BEGIN(soft_limit_reached) - llvm::Value* is_limit_reached_res = AddIsSoftMemoryLimitReached(ctx); - JIT_IF_EVAL(is_limit_reached_res) - IssueDebugLog("Soft memory limit reached"); - JIT_RETURN_CONST(MOT::RC_MEMORY_ALLOCATION_ERROR); - JIT_ELSE() - IssueDebugLog("Soft memory limit not reached"); - JIT_IF_END() -} - -/** @brief Builds a code segment for writing datum value to a column. */ -static void buildWriteDatumColumn(JitLlvmCodeGenContext* ctx, llvm::Value* row, int colid, llvm::Value* datum_value) -{ - llvm::Value* set_null_bit_res = AddSetExprResultNullBit(ctx, row, colid); - IssueDebugLog("Set null bit"); - - if (ctx->_table_info.m_table->GetField(colid)->m_isNotNull) { - JIT_IF_BEGIN(check_null_violation) - JIT_IF_EVAL_CMP(set_null_bit_res, JIT_CONST(MOT::RC_OK), JIT_ICMP_NE) - IssueDebugLog("Null constraint violated"); - JIT_RETURN(set_null_bit_res); - JIT_IF_END() - } - - // now check if the result is not null, and if so write column datum - llvm::Value* is_expr_null = AddGetExprArgIsNull(ctx, 0); - JIT_IF_BEGIN(check_expr_null) - JIT_IF_EVAL_NOT(is_expr_null) - IssueDebugLog("Encountered non-null expression result, writing datum column"); - AddWriteDatumColumn(ctx, colid, row, datum_value); - JIT_IF_END() -} - -/** @brief Builds a code segment for writing a row. */ -void buildWriteRow(JitLlvmCodeGenContext* ctx, llvm::Value* row, bool isPKey, JitLlvmRuntimeCursor* cursor) -{ - IssueDebugLog("Writing row"); - llvm::Value* write_row_res = AddWriteRow(ctx, row); - - JIT_IF_BEGIN(check_row_written) - JIT_IF_EVAL_CMP(write_row_res, JIT_CONST(MOT::RC_OK), JIT_ICMP_NE) - IssueDebugLog("Row not written"); - // need to emit cleanup code - if (!isPKey) { - AddDestroyCursor(ctx, cursor); - } - JIT_RETURN(write_row_res); - JIT_IF_END() -} - -/** @brief Process a join expression (WHERE clause) and generate code to build a search key. */ -static bool ProcessJoinExpr(JitLlvmCodeGenContext* ctx, Expr* expr, int* column_count, int* column_array, int* max_arg) -{ - bool result = false; - if (expr->type == T_OpExpr) { - result = ProcessJoinOpExpr(ctx, (OpExpr*)expr, column_count, column_array, max_arg); - } else if (expr->type == T_BoolExpr) { - result = ProcessJoinBoolExpr(ctx, (BoolExpr*)expr, column_count, column_array, max_arg); - } else { - MOT_LOG_TRACE("Unsupported expression type %d while processing Join Expr", (int)expr->type); - } - return result; -} - -/** @brief Process an operator expression (process only "COLUMN equals EXPR" operators). */ -static bool ProcessJoinOpExpr( - JitLlvmCodeGenContext* ctx, const OpExpr* op_expr, int* column_count, int* column_array, int* max_arg) -{ - bool result = false; - // process only point queries - if (IsWhereOperatorSupported(op_expr->opno)) { - llvm::Value* value = nullptr; - ListCell* lc1 = nullptr; - int colid = -1; - int vartype = -1; - int result_type = -1; - - foreach (lc1, op_expr->args) { - Expr* expr = (Expr*)lfirst(lc1); - // sometimes relabel expression hides the inner expression, so we peel it off - if (expr->type == T_RelabelType) { - expr = ((RelabelType*)expr)->arg; - } - if (expr->type == T_Var) { - Var* var = (Var*)expr; - colid = var->varattno; - vartype = var->vartype; - if (!IsTypeSupported(vartype)) { - MOT_LOG_TRACE("ProcessJoinOpExpr(): Unsupported type %d", vartype); - return false; + TxnAccess* tx = txMan->m_accessMgr; + TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); + auto itr = orderedSet.begin(); + while (itr != orderedSet.end()) { + Access* ac = (*itr).second; + switch (ac->m_type) { + case WR: + m_writeSetSize++; + gcMemoryReserve.m_version_queue++; + break; + case DEL: + m_writeSetSize++; + if (ac->m_params.IsPrimarySentinel()) { + gcMemoryReserve.m_delete_queue++; + } else { + if (ac->m_params.IsIndexUpdate()) { + gcMemoryReserve.m_update_column_queue++; + } } - // no further processing - } else { - value = ProcessExpr(ctx, expr, result_type, 0, 0, max_arg); - if (value == nullptr) { - MOT_LOG_TRACE("Unsupported operand type %d while processing Join OpExpr", (int)expr->type); - } - if (!IsTypeSupported(result_type)) { - MOT_LOG_TRACE("ProcessJoinOpExpr(): Unsupported result type %d", result_type); - return false; + gcMemoryReserve.m_generic_queue++; + break; + case INS: + m_insertSetSize++; + m_writeSetSize++; + if (ac->m_params.IsUpgradeInsert()) { + gcMemoryReserve.m_version_queue++; } break; - } - } - - if ((colid != -1) && (value != nullptr) && (vartype != -1) && (result_type != -1)) { - if (result_type != vartype) { - MOT_LOG_TRACE("ProcessJoinOpExpr(): vartype %d and result-type %d mismatch", vartype, result_type); - return false; - } - // execute: column = getColumnAt(colid) - llvm::Value* column = AddGetColumnAt(ctx, - colid, - JIT_RANGE_SCAN_MAIN); // no need to translate to zero-based index (first column is null bits) - int index_colid = ctx->_table_info.m_columnMap[colid]; - AddBuildDatumKey(ctx, column, index_colid, value, vartype, JIT_RANGE_ITERATOR_START, JIT_RANGE_SCAN_MAIN); - - MOT_LOG_DEBUG("Encountered table column %d, index column %d in where clause", colid, index_colid); - ++(*column_count); - column_array[index_colid] = 1; - result = true; - } else { - MOT_LOG_TRACE("ProcessJoinOpExpr(): Invalid expression (colid=%d, value=%p, vartype=%d, result_type=%d)", - colid, - value, - vartype, - result_type); - } - } else { - MOT_LOG_TRACE("ProcessJoinOpExpr(): Unsupported operator type %u", op_expr->opno); - } - - return result; -} - -/** @brief Process a boolean operator (process only AND operators, since we handle only point queries, or full-prefix - * range update). */ -static bool ProcessJoinBoolExpr( - JitLlvmCodeGenContext* ctx, const BoolExpr* boolexpr, int* column_count, int* column_array, int* max_arg) -{ - bool result = false; - if (boolexpr->boolop == AND_EXPR) { - // now traverse args to get param index to build search key - ListCell* lc = nullptr; - foreach (lc, boolexpr->args) { - // each element is Expr - Expr* expr = (Expr*)lfirst(lc); - result = ProcessJoinExpr(ctx, expr, column_count, column_array, max_arg); - if (!result) { - MOT_LOG_TRACE("Failed to process operand while processing Join BoolExpr"); - break; - } - } - } else { - MOT_LOG_TRACE("Unsupported bool operation %d while processing Join BoolExpr", (int)boolexpr->boolop); - } - return result; -} - -/** @brief Adds code to reset the number of rows processed. */ -void buildResetRowsProcessed(JitLlvmCodeGenContext* ctx) -{ - ctx->rows_processed = llvm::ConstantInt::get(ctx->INT64_T, 0, true); -} - -/** @brief Adds code to increment the number of rows processed. */ -void buildIncrementRowsProcessed(JitLlvmCodeGenContext* ctx) -{ - llvm::ConstantInt* one_value = llvm::ConstantInt::get(ctx->INT64_T, 1, true); - ctx->rows_processed = ctx->_builder->CreateAdd(ctx->rows_processed, one_value); -} - -/** @brief Adds code to create a new row. */ -llvm::Value* buildCreateNewRow(JitLlvmCodeGenContext* ctx) -{ - llvm::Value* row = AddCreateNewRow(ctx); - - JIT_IF_BEGIN(check_row_created) - JIT_IF_EVAL_NOT(row) - IssueDebugLog("Failed to create row"); - JIT_RETURN_CONST(MOT::RC_MEMORY_ALLOCATION_ERROR); - JIT_IF_END() - - return row; -} - -/** @brief Adds code to search for a row by a key. */ -llvm::Value* buildSearchRow(JitLlvmCodeGenContext* ctx, MOT::AccessType access_type, JitRangeScanType range_scan_type, - int subQueryIndex /* = -1 */) -{ - IssueDebugLog("Searching row"); - llvm::Value* row = AddSearchRow(ctx, access_type, range_scan_type, subQueryIndex); - - JIT_IF_BEGIN(check_row_found) - JIT_IF_EVAL_NOT(row) - IssueDebugLog("Row not found"); - JIT_RETURN_CONST(MOT::RC_LOCAL_ROW_NOT_FOUND); - JIT_IF_END() - - IssueDebugLog("Row found"); - return row; -} - -static llvm::Value* buildFilter(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitFilter* filter, int* max_arg) -{ - llvm::Value* result = nullptr; - llvm::Value* lhs_expr = ProcessExpr(ctx, row, filter->_lhs_operand, max_arg); - if (lhs_expr == nullptr) { - MOT_LOG_TRACE( - "buildFilter(): Failed to process LHS expression with type %d", (int)filter->_lhs_operand->_expr_type); - } else { - llvm::Value* rhs_expr = ProcessExpr(ctx, row, filter->_rhs_operand, max_arg); - if (rhs_expr == nullptr) { - MOT_LOG_TRACE( - "buildFilter(): Failed to process RHS expression with type %d", (int)filter->_rhs_operand->_expr_type); - } else { - result = ProcessFilterExpr(ctx, row, filter, max_arg); - } - } - return result; -} - -bool buildFilterRow( - JitLlvmCodeGenContext* ctx, llvm::Value* row, JitFilterArray* filters, int* max_arg, llvm::BasicBlock* next_block) -{ - // 1. for each filter expression we generate the equivalent instructions which should evaluate to true or false - // 2. We assume that all filters are applied with AND operator between them (imposed during query analysis/plan - // phase) - for (int i = 0; i < filters->_filter_count; ++i) { - llvm::Value* filter_expr = buildFilter(ctx, row, &filters->_scan_filters[i], max_arg); - if (filter_expr == nullptr) { - MOT_LOG_TRACE("buildFilterRow(): Failed to process filter expression %d", i); - return false; - } - - JIT_IF_BEGIN(filter_row) - JIT_IF_EVAL_NOT(filter_expr) - IssueDebugLog("Row filter expression failed"); - if (next_block == nullptr) { - JIT_RETURN_CONST(MOT::RC_LOCAL_ROW_NOT_FOUND); - } else { - ctx->_builder->CreateBr(next_block); - } - JIT_IF_END() - } - return true; -} - -/** @brief Adds code to insert a new row. */ -void buildInsertRow(JitLlvmCodeGenContext* ctx, llvm::Value* row) -{ - IssueDebugLog("Inserting row"); - llvm::Value* insert_row_res = AddInsertRow(ctx, row); - - JIT_IF_BEGIN(check_row_inserted) - JIT_IF_EVAL_CMP(insert_row_res, JIT_CONST(MOT::RC_OK), JIT_ICMP_NE) - IssueDebugLog("Row not inserted"); - JIT_RETURN(insert_row_res); - JIT_IF_END() - - IssueDebugLog("Row inserted"); -} - -/** @brief Adds code to delete a row. */ -void buildDeleteRow(JitLlvmCodeGenContext* ctx) -{ - IssueDebugLog("Deleting row"); - llvm::Value* delete_row_res = AddDeleteRow(ctx); - - JIT_IF_BEGIN(check_delete_row) - JIT_IF_EVAL_CMP(delete_row_res, JIT_CONST(MOT::RC_OK), JIT_ICMP_NE) - IssueDebugLog("Row not deleted"); - JIT_RETURN(delete_row_res); - JIT_IF_END() - - IssueDebugLog("Row deleted"); -} - -/** @brief Adds code to search for an iterator. */ -static llvm::Value* buildSearchIterator(JitLlvmCodeGenContext* ctx, JitIndexScanDirection index_scan_direction, - JitRangeBoundMode range_bound_mode, JitRangeScanType range_scan_type, int subQueryIndex = -1) -{ - // search the row - IssueDebugLog("Searching range start"); - llvm::Value* itr = AddSearchIterator(ctx, index_scan_direction, range_bound_mode, range_scan_type, subQueryIndex); - - JIT_IF_BEGIN(check_itr_found) - JIT_IF_EVAL_NOT(itr) - IssueDebugLog("Range start not found"); - JIT_RETURN_CONST(MOT::RC_LOCAL_ROW_NOT_FOUND); - JIT_IF_END() - - IssueDebugLog("Range start found"); - return itr; -} - -/** @brief Adds code to search for an iterator. */ -static llvm::Value* buildBeginIterator( - JitLlvmCodeGenContext* ctx, JitRangeScanType rangeScanType, int subQueryIndex = -1) -{ - // search the row - IssueDebugLog("Getting begin iterator for full-scan"); - llvm::Value* itr = AddBeginIterator(ctx, rangeScanType, subQueryIndex); - - JIT_IF_BEGIN(check_itr_found) - JIT_IF_EVAL_NOT(itr) - IssueDebugLog("Begin iterator not found"); - JIT_RETURN_CONST(MOT::RC_LOCAL_ROW_NOT_FOUND); - JIT_IF_END() - - IssueDebugLog("Range start found"); - return itr; -} - -/** @brief Adds code to get row from iterator. */ -llvm::Value* buildGetRowFromIterator(JitLlvmCodeGenContext* ctx, llvm::BasicBlock* endLoopBlock, - MOT::AccessType access_mode, JitIndexScanDirection index_scan_direction, JitLlvmRuntimeCursor* cursor, - JitRangeScanType range_scan_type, int subQueryIndex /* = -1 */) -{ - IssueDebugLog("Retrieving row from iterator"); - llvm::Value* row = - AddGetRowFromIterator(ctx, access_mode, index_scan_direction, cursor, range_scan_type, subQueryIndex); - - JIT_IF_BEGIN(check_itr_row_found) - JIT_IF_EVAL_NOT(row) - IssueDebugLog("Iterator row not found"); - JIT_GOTO(endLoopBlock); - // NOTE: we can actually do here a JIT_WHILE_BREAK() if we have an enclosing while loop - JIT_IF_END() - - IssueDebugLog("Iterator row found"); - return row; -} - -/** @brief Process constant expression. */ -static llvm::Value* ProcessConstExpr( - JitLlvmCodeGenContext* ctx, const Const* const_value, int& result_type, int arg_pos, int depth, int* max_arg) -{ - llvm::Value* result = nullptr; - MOT_LOG_DEBUG("%*s --> Processing CONST expression", depth, ""); - if (depth > MOT_JIT_MAX_EXPR_DEPTH) { - MOT_LOG_TRACE("Cannot process expression: Expression exceeds depth limit %d", (int)MOT_JIT_MAX_EXPR_DEPTH); - return nullptr; - } - - // verify type is supported and return compile-time constant (no need to generate code for runtime evaluation) - if (IsTypeSupported(const_value->consttype)) { - result_type = const_value->consttype; - AddSetExprArgIsNull(ctx, arg_pos, const_value->constisnull); // mark expression null status - if (IsPrimitiveType(result_type)) { - result = llvm::ConstantInt::get(ctx->INT64_T, const_value->constvalue, true); - } else { - int constId = AllocateConstId(ctx, result_type, const_value->constvalue, const_value->constisnull); - if (constId == -1) { - MOT_LOG_TRACE("Failed to allocate constant identifier"); - } else { - result = AddGetConstAt(ctx, constId, arg_pos); - } - } - if (max_arg && (arg_pos > *max_arg)) { - *max_arg = arg_pos; - } - } else { - MOT_LOG_TRACE("Failed to process const expression: type %d unsupported", (int)result_type); - } - - MOT_LOG_DEBUG("%*s <-- Processing CONST expression result: %p", depth, "", result); - return result; -} - -/** @brief Process Param expression. */ -static llvm::Value* ProcessParamExpr( - JitLlvmCodeGenContext* ctx, const Param* param, int& result_type, int arg_pos, int depth, int* max_arg) -{ - llvm::Value* result = nullptr; - MOT_LOG_DEBUG("%*s --> Processing PARAM expression", depth, ""); - if (depth > MOT_JIT_MAX_EXPR_DEPTH) { - MOT_LOG_TRACE("Cannot process expression: Expression exceeds depth limit %d", (int)MOT_JIT_MAX_EXPR_DEPTH); - return nullptr; - } - - // verify type is supported and generate code to extract the parameter in runtime - if (IsTypeSupported(param->paramtype)) { - result_type = param->paramtype; - result = AddGetDatumParam(ctx, param->paramid - 1, arg_pos); - if (max_arg && (arg_pos > *max_arg)) { - *max_arg = arg_pos; - } - } - - MOT_LOG_DEBUG("%*s <-- Processing PARAM expression result: %p", depth, "", result); - return result; -} - -/** @brief Process Relabel expression as Param expression. */ -static llvm::Value* ProcessRelabelExpr( - JitLlvmCodeGenContext* ctx, RelabelType* relabel_type, int& result_type, int arg_pos, int depth, int* max_arg) -{ - llvm::Value* result = nullptr; - MOT_LOG_DEBUG("Processing RELABEL expression"); - Expr* expr = (Expr*)relabel_type->arg; - if (expr->type == T_Param) { - Param* param = (Param*)expr; - result = ProcessParamExpr(ctx, param, result_type, arg_pos, depth, max_arg); - } else { - MOT_LOG_TRACE("Unexpected relabel argument type: %d", (int)expr->type); - } - return result; -} - -/** @brief Proess Var expression. */ -static llvm::Value* ProcessVarExpr( - JitLlvmCodeGenContext* ctx, const Var* var, int& result_type, int arg_pos, int depth, int* max_arg) -{ - llvm::Value* result = nullptr; - MOT_LOG_DEBUG("%*s --> Processing VAR expression", depth, ""); - if (depth > MOT_JIT_MAX_EXPR_DEPTH) { - MOT_LOG_TRACE("Cannot process expression: Expression exceeds depth limit %d", (int)MOT_JIT_MAX_EXPR_DEPTH); - return nullptr; - } - - // verify that type is supported and generate code to read column datum during runtime - if (IsTypeSupported(var->vartype)) { - result_type = var->vartype; - int table_colid = var->varattno; - result = AddReadDatumColumn(ctx, ctx->table_value, nullptr, table_colid, arg_pos); - if (max_arg && (arg_pos > *max_arg)) { - *max_arg = arg_pos; - } - } - - MOT_LOG_DEBUG("%*s <-- Processing VAR expression result: %p", depth, "", result); - return result; -} - -/** @brief Adds call to PG unary operator. */ -static llvm::Value* AddExecUnaryOperator( - JitLlvmCodeGenContext* ctx, llvm::Value* param, llvm::FunctionCallee unary_operator, int arg_pos) -{ - llvm::Constant* arg_pos_value = llvm::ConstantInt::get(ctx->INT32_T, arg_pos, true); - return AddFunctionCall(ctx, unary_operator, param, arg_pos_value, nullptr); -} - -/** @brief Adds call to PG binary operator. */ -static llvm::Value* AddExecBinaryOperator(JitLlvmCodeGenContext* ctx, llvm::Value* lhs_param, llvm::Value* rhs_param, - llvm::FunctionCallee binary_operator, int arg_pos) -{ - llvm::Constant* arg_pos_value = llvm::ConstantInt::get(ctx->INT32_T, arg_pos, true); - return AddFunctionCall(ctx, binary_operator, lhs_param, rhs_param, arg_pos_value, nullptr); -} - -/** @brief Adds call to PG ternary operator. */ -static llvm::Value* AddExecTernaryOperator(JitLlvmCodeGenContext* ctx, llvm::Value* param1, llvm::Value* param2, - llvm::Value* param3, llvm::FunctionCallee ternary_operator, int arg_pos) -{ - llvm::Constant* arg_pos_value = llvm::ConstantInt::get(ctx->INT32_T, arg_pos, true); - return AddFunctionCall(ctx, ternary_operator, param1, param2, param3, arg_pos_value, nullptr); -} - -#define APPLY_UNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_DEBUG("Adding call to builtin: " #name); \ - result = AddExecUnaryOperator(ctx, args[0], ctx->_builtin_##name, arg_pos); \ - break; - -#define APPLY_BINARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_DEBUG("Adding call to builtin: " #name); \ - result = AddExecBinaryOperator(ctx, args[0], args[1], ctx->_builtin_##name, arg_pos); \ - break; - -#define APPLY_TERNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_DEBUG("Adding call to builtin: " #name); \ - result = AddExecTernaryOperator(ctx, args[0], args[1], args[2], ctx->_builtin_##name, arg_pos); \ - break; - -#define APPLY_UNARY_CAST_OPERATOR(funcid, name) APPLY_UNARY_OPERATOR(funcid, name) -#define APPLY_BINARY_CAST_OPERATOR(funcid, name) APPLY_BINARY_OPERATOR(funcid, name) -#define APPLY_TERNARY_CAST_OPERATOR(funcid, name) APPLY_TERNARY_OPERATOR(funcid, name) - -/** @brief Process operator expression. */ -static llvm::Value* ProcessOpExpr( - JitLlvmCodeGenContext* ctx, const OpExpr* op_expr, int& result_type, int arg_pos, int depth, int* max_arg) -{ - llvm::Value* result = nullptr; - MOT_LOG_DEBUG("%*s --> Processing OP %u expression", depth, "", op_expr->opfuncid); - if (depth > MOT_JIT_MAX_EXPR_DEPTH) { - MOT_LOG_TRACE("Cannot process expression: Expression exceeds depth limit %d", (int)MOT_JIT_MAX_EXPR_DEPTH); - return nullptr; - } - - if (list_length(op_expr->args) > 3) { - MOT_LOG_TRACE("Unsupported operator %u: too many arguments", op_expr->opno); - return nullptr; - } - - llvm::Value* args[3] = {nullptr, nullptr, nullptr}; - int arg_num = 0; - int dummy = 0; - - // process operator arguments (each one is an expression by itself) - ListCell* lc = nullptr; - foreach (lc, op_expr->args) { - Expr* sub_expr = (Expr*)lfirst(lc); - args[arg_num] = ProcessExpr(ctx, sub_expr, dummy, arg_pos + arg_num, depth + 1, max_arg); - if (args[arg_num] == nullptr) { - MOT_LOG_TRACE("Failed to process operator sub-expression %d", arg_num); - return nullptr; - } - if (++arg_num == 3) { - break; - } - } - - // process the operator - generate code to call the operator in runtime - result_type = op_expr->opresulttype; - switch (op_expr->opfuncid) { - APPLY_OPERATORS() - - default: - MOT_LOG_TRACE("Unsupported operator function type: %u", op_expr->opfuncid); - break; - } - - MOT_LOG_DEBUG("%*s <-- Processing OP %u expression result: %p", depth, "", op_expr->opfuncid, result); - return result; -} - -/** @brief Process function expression. */ -static llvm::Value* ProcessFuncExpr( - JitLlvmCodeGenContext* ctx, const FuncExpr* func_expr, int& result_type, int arg_pos, int depth, int* max_arg) -{ - llvm::Value* result = nullptr; - MOT_LOG_DEBUG("%*s --> Processing FUNC %d expression", depth, "", (int)func_expr->funcid); - if (depth > MOT_JIT_MAX_EXPR_DEPTH) { - MOT_LOG_TRACE("Cannot process expression: Expression exceeds depth limit %d", (int)MOT_JIT_MAX_EXPR_DEPTH); - return nullptr; - } - - if (list_length(func_expr->args) > 3) { - MOT_LOG_TRACE("Unsupported function %d: too many arguments", func_expr->funcid); - return nullptr; - } - - llvm::Value* args[3] = {nullptr, nullptr, nullptr}; - int arg_num = 0; - int dummy = 0; - - // process function arguments (each one is an expression by itself) - ListCell* lc = nullptr; - foreach (lc, func_expr->args) { - Expr* sub_expr = (Expr*)lfirst(lc); - args[arg_num] = ProcessExpr(ctx, sub_expr, dummy, arg_pos + arg_num, depth + 1, max_arg); - if (args[arg_num] == nullptr) { - MOT_LOG_TRACE("Failed to process function sub-expression %d", arg_num); - return nullptr; - } - if (++arg_num == 3) { - break; - } - } - - // process the function - generate code to call the function in runtime - result_type = func_expr->funcresulttype; - switch (func_expr->funcid) { - APPLY_OPERATORS() - - default: - MOT_LOG_TRACE("Unsupported function type: %d", (int)func_expr->funcid); - break; - } - - MOT_LOG_DEBUG("%*s <-- Processing FUNC %d expression result: %p", depth, "", (int)func_expr->funcid, result); - return result; -} - -// we allow only binary operators for filters -#undef APPLY_UNARY_OPERATOR -#undef APPLY_TERNARY_OPERATOR -#undef APPLY_UNARY_CAST_OPERATOR -#undef APPLY_TERNARY_CAST_OPERATOR - -#define APPLY_UNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_TRACE("Unexpected call in filter expression to unary builtin: " #name); \ - break; - -#define APPLY_TERNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_TRACE("Unexpected call in filter expression to ternary builtin: " #name); \ - break; - -#define APPLY_UNARY_CAST_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_TRACE("Unexpected call in filter expression to unary cast builtin: " #name); \ - break; - -#define APPLY_TERNARY_CAST_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_TRACE("Unexpected call in filter expression to ternary cast builtin: " #name); \ - break; - -static llvm::Value* ProcessFilterExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitFilter* filter, int* max_arg) -{ - llvm::Value* result = nullptr; - - llvm::Value* args[3] = {nullptr, nullptr, nullptr}; - - args[0] = ProcessExpr(ctx, row, filter->_lhs_operand, max_arg); - if (!args[0]) { - MOT_LOG_TRACE("Failed to process filter LHS expression"); - return nullptr; - } - - args[1] = ProcessExpr(ctx, row, filter->_rhs_operand, max_arg); - if (!args[1]) { - MOT_LOG_TRACE("Failed to process filter RHS expression"); - return nullptr; - } - - int arg_pos = 0; // always a top-level expression - switch (filter->_filter_op_funcid) { - APPLY_OPERATORS() - - default: - MOT_LOG_TRACE("Unsupported filter function type: %d", filter->_filter_op_funcid); - break; - } - - return result; -} - -#undef APPLY_UNARY_OPERATOR -#undef APPLY_BINARY_OPERATOR -#undef APPLY_TERNARY_OPERATOR -#undef APPLY_UNARY_CAST_OPERATOR -#undef APPLY_BINARY_CAST_OPERATOR -#undef APPLY_TERNARY_CAST_OPERATOR - -/** @brief Process an expression. Generates code to evaluate the expression. */ -static llvm::Value* ProcessExpr( - JitLlvmCodeGenContext* ctx, Expr* expr, int& result_type, int arg_pos, int depth, int* max_arg) -{ - llvm::Value* result = nullptr; - MOT_LOG_DEBUG("%*s --> Processing expression %d", depth, "", (int)expr->type); - if (depth > MOT_JIT_MAX_EXPR_DEPTH) { - MOT_LOG_TRACE("Cannot process expression: Expression exceeds depth limit %d", (int)MOT_JIT_MAX_EXPR_DEPTH); - return nullptr; - } - - // case 1: assign from parameter (cases like: s_quantity = $1) - if (expr->type == T_Const) { - result = ProcessConstExpr(ctx, (Const*)expr, result_type, arg_pos, depth + 1, max_arg); - } else if (expr->type == T_Param) { - result = ProcessParamExpr(ctx, (Param*)expr, result_type, arg_pos, depth + 1, max_arg); - } else if (expr->type == T_RelabelType) { - result = ProcessRelabelExpr(ctx, (RelabelType*)expr, result_type, arg_pos, depth + 1, max_arg); - } else if (expr->type == T_Var) { - result = ProcessVarExpr(ctx, (Var*)expr, result_type, arg_pos, depth + 1, max_arg); - } else if (expr->type == T_OpExpr) { - result = ProcessOpExpr(ctx, (OpExpr*)expr, result_type, arg_pos, depth + 1, max_arg); - } else if (expr->type == T_FuncExpr) { - result = ProcessFuncExpr(ctx, (FuncExpr*)expr, result_type, arg_pos, depth + 1, max_arg); - } else { - MOT_LOG_TRACE( - "Failed to generate jitted code for query: unsupported target expression type: %d", (int)expr->type); - } - - MOT_LOG_DEBUG("%*s <-- Processing expression %d result: %p", depth, "", (int)expr->type, result); - return result; -} - -static llvm::Value* ProcessConstExpr(JitLlvmCodeGenContext* ctx, const JitConstExpr* expr, int* max_arg) -{ - llvm::Value* result = nullptr; - AddSetExprArgIsNull(ctx, expr->_arg_pos, expr->_is_null); // mark expression null status - if (IsPrimitiveType(expr->_const_type)) { - result = llvm::ConstantInt::get(ctx->INT64_T, expr->_value, true); - } else { - int constId = AllocateConstId(ctx, expr->_const_type, expr->_value, expr->_is_null); - if (constId == -1) { - MOT_LOG_TRACE("Failed to allocate constant identifier"); - } else { - result = AddGetConstAt(ctx, constId, expr->_arg_pos); - } - } - if (max_arg && (expr->_arg_pos > *max_arg)) { - *max_arg = expr->_arg_pos; - } - return result; -} - -static llvm::Value* ProcessParamExpr(JitLlvmCodeGenContext* ctx, const JitParamExpr* expr, int* max_arg) -{ - llvm::Value* result = AddGetDatumParam(ctx, expr->_param_id, expr->_arg_pos); - if (max_arg && (expr->_arg_pos > *max_arg)) { - *max_arg = expr->_arg_pos; - } - return result; -} - -static llvm::Value* ProcessVarExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, const JitVarExpr* expr, int* max_arg) -{ - llvm::Value* result = nullptr; - if (row == nullptr) { - MOT_LOG_TRACE("ProcessVarExpr(): Unexpected VAR expression without a row"); - } else { - // this is a bit awkward, but it works - llvm::Value* table = (expr->_table == ctx->_table_info.m_table) ? ctx->table_value : ctx->inner_table_value; - result = AddReadDatumColumn(ctx, table, row, expr->_column_id, expr->_arg_pos); - if (max_arg && (expr->_arg_pos > *max_arg)) { - *max_arg = expr->_arg_pos; - } - } - return result; -} - -#define APPLY_UNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_DEBUG("Adding call to builtin: " #name); \ - result = AddExecUnaryOperator(ctx, args[0], ctx->_builtin_##name, expr->_arg_pos); \ - break; - -#define APPLY_BINARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_DEBUG("Adding call to builtin: " #name); \ - result = AddExecBinaryOperator(ctx, args[0], args[1], ctx->_builtin_##name, expr->_arg_pos); \ - break; - -#define APPLY_TERNARY_OPERATOR(funcid, name) \ - case funcid: \ - MOT_LOG_DEBUG("Adding call to builtin: " #name); \ - result = AddExecTernaryOperator(ctx, args[0], args[1], args[2], ctx->_builtin_##name, expr->_arg_pos); \ - break; - -#define APPLY_UNARY_CAST_OPERATOR(funcid, name) APPLY_UNARY_OPERATOR(funcid, name) -#define APPLY_BINARY_CAST_OPERATOR(funcid, name) APPLY_BINARY_OPERATOR(funcid, name) -#define APPLY_TERNARY_CAST_OPERATOR(funcid, name) APPLY_TERNARY_OPERATOR(funcid, name) - -static llvm::Value* ProcessOpExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitOpExpr* expr, int* max_arg) -{ - llvm::Value* result = nullptr; - - llvm::Value* args[MOT_JIT_MAX_FUNC_EXPR_ARGS] = {nullptr, nullptr, nullptr}; - int arg_num = 0; - - for (int i = 0; i < expr->_arg_count; ++i) { - args[i] = ProcessExpr(ctx, row, expr->_args[i], max_arg); - if (args[i] == nullptr) { - MOT_LOG_TRACE("Failed to process operator sub-expression %d", arg_num); - return nullptr; - } - } - - switch (expr->_op_func_id) { - APPLY_OPERATORS() - - default: - MOT_LOG_TRACE("Unsupported operator function type: %d", expr->_op_func_id); - break; - } - - return result; -} - -static llvm::Value* ProcessFuncExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitFuncExpr* expr, int* max_arg) -{ - llvm::Value* result = nullptr; - - llvm::Value* args[MOT_JIT_MAX_FUNC_EXPR_ARGS] = {nullptr, nullptr, nullptr}; - int arg_num = 0; - - for (int i = 0; i < expr->_arg_count; ++i) { - args[i] = ProcessExpr(ctx, row, expr->_args[i], max_arg); - if (args[i] == nullptr) { - MOT_LOG_TRACE("Failed to process function sub-expression %d", arg_num); - return nullptr; - } - } - - switch (expr->_func_id) { - APPLY_OPERATORS() - - default: - MOT_LOG_TRACE("Unsupported function type: %d", expr->_func_id); - break; - } - - return result; -} - -#undef APPLY_UNARY_OPERATOR -#undef APPLY_BINARY_OPERATOR -#undef APPLY_TERNARY_OPERATOR -#undef APPLY_UNARY_CAST_OPERATOR -#undef APPLY_BINARY_CAST_OPERATOR -#undef APPLY_TERNARY_CAST_OPERATOR - -static llvm::Value* ProcessSubLinkExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitSubLinkExpr* expr, int* max_arg) -{ - return AddSelectSubQueryResult(ctx, expr->_sub_query_index); -} - -static llvm::Value* ProcessBoolExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitBoolExpr* expr, int* maxArg) -{ - llvm::Value* result = nullptr; - - llvm::Value* args[MOT_JIT_MAX_BOOL_EXPR_ARGS] = {nullptr, nullptr}; - int argNum = 0; - - for (int i = 0; i < expr->_arg_count; ++i) { - args[i] = ProcessExpr(ctx, row, expr->_args[i], maxArg); - if (args[i] == nullptr) { - MOT_LOG_TRACE("Failed to process boolean sub-expression %d", argNum); - return nullptr; - } - } - - llvm::Value* typedZero = llvm::ConstantInt::get(args[0]->getType(), 0, true); - switch (expr->_bool_expr_type) { - case NOT_EXPR: { - llvm::Value* notResult = ctx->_builder->CreateICmpEQ(args[0], typedZero); // equivalent to NOT - result = ctx->_builder->CreateIntCast(notResult, args[0]->getType(), true); - break; - } - - case AND_EXPR: - result = ctx->_builder->CreateSelect(args[0], args[1], typedZero); - break; - - case OR_EXPR: { - llvm::Value* typedOne = llvm::ConstantInt::get(args[0]->getType(), 1, true); - result = ctx->_builder->CreateSelect(args[0], typedOne, args[1]); - break; - } - - default: - MOT_LOG_TRACE("Unsupported boolean expression type: %d", (int)expr->_bool_expr_type); - break; - } - - return result; -} - -static llvm::Value* ProcessExpr(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitExpr* expr, int* max_arg) -{ - llvm::Value* result = nullptr; - - if (expr->_expr_type == JIT_EXPR_TYPE_CONST) { - result = ProcessConstExpr(ctx, (JitConstExpr*)expr, max_arg); - } else if (expr->_expr_type == JIT_EXPR_TYPE_PARAM) { - result = ProcessParamExpr(ctx, (JitParamExpr*)expr, max_arg); - } else if (expr->_expr_type == JIT_EXPR_TYPE_VAR) { - result = ProcessVarExpr(ctx, row, (JitVarExpr*)expr, max_arg); - } else if (expr->_expr_type == JIT_EXPR_TYPE_OP) { - result = ProcessOpExpr(ctx, row, (JitOpExpr*)expr, max_arg); - } else if (expr->_expr_type == JIT_EXPR_TYPE_FUNC) { - result = ProcessFuncExpr(ctx, row, (JitFuncExpr*)expr, max_arg); - } else if (expr->_expr_type == JIT_EXPR_TYPE_SUBLINK) { - result = ProcessSubLinkExpr(ctx, row, (JitSubLinkExpr*)expr, max_arg); - } else if (expr->_expr_type == JIT_EXPR_TYPE_BOOL) { - result = ProcessBoolExpr(ctx, row, (JitBoolExpr*)expr, max_arg); - } else { - MOT_LOG_TRACE( - "Failed to generate jitted code for query: unsupported target expression type: %d", (int)expr->_expr_type); - } - - return result; -} - -bool buildScanExpression(JitLlvmCodeGenContext* ctx, JitColumnExpr* expr, int* max_arg, - JitRangeIteratorType range_itr_type, JitRangeScanType range_scan_type, llvm::Value* outer_row, int subQueryIndex) -{ - llvm::Value* value = ProcessExpr(ctx, outer_row, expr->_expr, max_arg); - if (value == nullptr) { - MOT_LOG_TRACE("buildScanExpression(): Failed to process expression with type %d", (int)expr->_expr->_expr_type); - return false; - } else { - llvm::Value* column = AddGetColumnAt(ctx, - expr->_table_column_id, - range_scan_type, // no need to translate to zero-based index (first column is null bits) - subQueryIndex); - int index_colid = -1; - if (range_scan_type == JIT_RANGE_SCAN_INNER) { - index_colid = ctx->_inner_table_info.m_columnMap[expr->_table_column_id]; - } else if (range_scan_type == JIT_RANGE_SCAN_MAIN) { - index_colid = ctx->_table_info.m_columnMap[expr->_table_column_id]; - } else if (range_scan_type == JIT_RANGE_SCAN_SUB_QUERY) { - index_colid = ctx->m_subQueryTableInfo[subQueryIndex].m_columnMap[expr->_table_column_id]; - } - AddBuildDatumKey( - ctx, column, index_colid, value, expr->_column_type, range_itr_type, range_scan_type, subQueryIndex); - } - return true; -} - -bool buildPointScan(JitLlvmCodeGenContext* ctx, JitColumnExprArray* exprArray, int* maxArg, - JitRangeScanType rangeScanType, llvm::Value* outerRow, int exprCount /* = -1 */, int subQueryIndex /* = -1 */) -{ - if (exprCount == -1) { - exprCount = exprArray->_count; - } - AddInitSearchKey(ctx, rangeScanType, subQueryIndex); - for (int i = 0; i < exprCount; ++i) { - JitColumnExpr* expr = &exprArray->_exprs[i]; - - // validate the expression refers to the right table (in search expressions array, all expressions refer to the - // same table) - if (rangeScanType == JIT_RANGE_SCAN_INNER) { - if (expr->_table != ctx->_inner_table_info.m_table) { - MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, - "Generate LLVM JIT Code", - "Invalid expression table (expected inner table %s, got %s)", - ctx->_inner_table_info.m_table->GetTableName().c_str(), - expr->_table->GetTableName().c_str()); - return false; - } - } else if (rangeScanType == JIT_RANGE_SCAN_MAIN) { - if (expr->_table != ctx->_table_info.m_table) { - MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, - "Generate LLVM JIT Code", - "Invalid expression table (expected main/outer table %s, got %s)", - ctx->_table_info.m_table->GetTableName().c_str(), - expr->_table->GetTableName().c_str()); - return false; - } - } else if (rangeScanType == JIT_RANGE_SCAN_SUB_QUERY) { - if (expr->_table != ctx->m_subQueryTableInfo[subQueryIndex].m_table) { - MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, - "Generate TVM JIT Code", - "Invalid expression table (expected sub-query table %s, got %s)", - ctx->m_subQueryTableInfo[subQueryIndex].m_table->GetTableName().c_str(), - expr->_table->GetTableName().c_str()); - return false; - } - } - - // prepare the sub-expression - if (!buildScanExpression(ctx, expr, maxArg, JIT_RANGE_ITERATOR_START, rangeScanType, outerRow, subQueryIndex)) { - return false; - } - } - return true; -} - -bool writeRowColumns( - JitLlvmCodeGenContext* ctx, llvm::Value* row, JitColumnExprArray* expr_array, int* max_arg, bool is_update) -{ - for (int i = 0; i < expr_array->_count; ++i) { - JitColumnExpr* column_expr = &expr_array->_exprs[i]; - - llvm::Value* value = ProcessExpr(ctx, row, column_expr->_expr, max_arg); - if (value == nullptr) { - MOT_LOG_TRACE("ProcessExpr() returned nullptr"); - return false; - } - - // set null bit or copy result data to column - buildWriteDatumColumn(ctx, row, column_expr->_table_column_id, value); - - // set bit for incremental redo - if (is_update) { - AddSetBit(ctx, column_expr->_table_column_id - 1); - } - } - - return true; -} - -bool selectRowColumns(JitLlvmCodeGenContext* ctx, llvm::Value* row, JitSelectExprArray* expr_array, int* max_arg, - JitRangeScanType range_scan_type, int subQueryIndex /* = -1 */) -{ - bool result = true; - for (int i = 0; i < expr_array->_count; ++i) { - JitSelectExpr* expr = &expr_array->_exprs[i]; - // we skip expressions that select from other tables - if (range_scan_type == JIT_RANGE_SCAN_INNER) { - if (expr->_column_expr->_table != ctx->_inner_table_info.m_table) { + case RD_FOR_UPDATE: + case RD: + itr = orderedSet.erase(itr); + txMan->m_accessMgr->PubReleaseAccess(ac); continue; - } - } else if (range_scan_type == JIT_RANGE_SCAN_MAIN) { - if (expr->_column_expr->_table != ctx->_table_info.m_table) { - continue; - } - } else if (range_scan_type == JIT_RANGE_SCAN_SUB_QUERY) { - if (expr->_column_expr->_table != ctx->m_subQueryTableInfo[subQueryIndex].m_table) { - continue; - } - } - result = AddSelectColumn( - ctx, row, expr->_column_expr->_column_id, expr->_tuple_column_id, range_scan_type, subQueryIndex); - if (!result) { - break; - } - } - - return result; -} - -static bool buildClosedRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* index_scan, int* max_arg, - JitRangeScanType range_scan_type, llvm::Value* outer_row, int subQueryIndex) -{ - // a closed range scan starts just like a point scan (without enough search expressions) and then adds key patterns - bool result = - buildPointScan(ctx, &index_scan->_search_exprs, max_arg, range_scan_type, outer_row, -1, subQueryIndex); - if (result) { - AddCopyKey(ctx, range_scan_type, subQueryIndex); - - // now fill each key with the right pattern for the missing pkey columns in the where clause - bool ascending = (index_scan->_sort_order == JIT_QUERY_SORT_ASCENDING); - - int* index_column_offsets = nullptr; - const uint16_t* key_length = nullptr; - int index_column_count = 0; - if (range_scan_type == JIT_RANGE_SCAN_INNER) { - index_column_offsets = ctx->_inner_table_info.m_indexColumnOffsets; - key_length = ctx->_inner_table_info.m_index->GetLengthKeyFields(); - index_column_count = ctx->_inner_table_info.m_index->GetNumFields(); - } else if (range_scan_type == JIT_RANGE_SCAN_MAIN) { - index_column_offsets = ctx->_table_info.m_indexColumnOffsets; - key_length = ctx->_table_info.m_index->GetLengthKeyFields(); - index_column_count = ctx->_table_info.m_index->GetNumFields(); - } else if (range_scan_type == JIT_RANGE_SCAN_SUB_QUERY) { - index_column_offsets = ctx->m_subQueryTableInfo[subQueryIndex].m_indexColumnOffsets; - key_length = ctx->m_subQueryTableInfo[subQueryIndex].m_index->GetLengthKeyFields(); - index_column_count = ctx->m_subQueryTableInfo[subQueryIndex].m_index->GetNumFields(); - } - - int first_zero_column = index_scan->_column_count; - for (int i = first_zero_column; i < index_column_count; ++i) { - int offset = index_column_offsets[i]; - int size = key_length[i]; - MOT_LOG_DEBUG( - "Filling begin/end iterator pattern for missing pkey fields at offset %d, size %d", offset, size); - AddFillKeyPattern( - ctx, ascending ? 0x00 : 0xFF, offset, size, JIT_RANGE_ITERATOR_START, range_scan_type, subQueryIndex); - AddFillKeyPattern( - ctx, ascending ? 0xFF : 0x00, offset, size, JIT_RANGE_ITERATOR_END, range_scan_type, subQueryIndex); - } - - AddAdjustKey(ctx, - ascending ? 0x00 : 0xFF, - JIT_RANGE_ITERATOR_START, - range_scan_type, // currently this is relevant only for secondary index searches - subQueryIndex); - AddAdjustKey(ctx, - ascending ? 0xFF : 0x00, - JIT_RANGE_ITERATOR_END, - range_scan_type, // currently this is relevant only for secondary index searches - subQueryIndex); - } - return result; -} - -static void BuildAscendingSemiOpenRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* indexScan, int* maxArg, - JitRangeScanType rangeScanType, JitRangeBoundMode* beginRangeBound, JitRangeBoundMode* endRangeBound, - llvm::Value* outerRow, int subQueryIndex, int offset, int size, JitColumnExpr* lastExpr) -{ - if ((indexScan->_last_dim_op1 == JIT_WOC_LESS_THAN) || (indexScan->_last_dim_op1 == JIT_WOC_LESS_EQUALS)) { - // this is an upper bound operator on an ascending semi-open scan so we fill the begin key with zeros, - // and the end key with the value - AddFillKeyPattern(ctx, 0x00, offset, size, JIT_RANGE_ITERATOR_START, rangeScanType, subQueryIndex); - buildScanExpression(ctx, lastExpr, maxArg, JIT_RANGE_ITERATOR_END, rangeScanType, outerRow, subQueryIndex); - *beginRangeBound = JIT_RANGE_BOUND_INCLUDE; - *endRangeBound = - (indexScan->_last_dim_op1 == JIT_WOC_LESS_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - } else { - // this is a lower bound operator on an ascending semi-open scan so we fill the begin key with the - // value, and the end key with 0xFF - buildScanExpression(ctx, lastExpr, maxArg, JIT_RANGE_ITERATOR_START, rangeScanType, outerRow, subQueryIndex); - AddFillKeyPattern(ctx, 0xFF, offset, size, JIT_RANGE_ITERATOR_END, rangeScanType, subQueryIndex); - *beginRangeBound = - (indexScan->_last_dim_op1 == JIT_WOC_GREATER_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - *endRangeBound = JIT_RANGE_BOUND_INCLUDE; - } -} - -static void BuildDescendingSemiOpenRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* indexScan, int* maxArg, - JitRangeScanType rangeScanType, JitRangeBoundMode* beginRangeBound, JitRangeBoundMode* endRangeBound, - llvm::Value* outerRow, int subQueryIndex, int offset, int size, JitColumnExpr* lastExpr) -{ - if ((indexScan->_last_dim_op1 == JIT_WOC_LESS_THAN) || (indexScan->_last_dim_op1 == JIT_WOC_LESS_EQUALS)) { - // this is an upper bound operator on a descending semi-open scan so we fill the begin key with value, - // and the end key with zeroes - buildScanExpression(ctx, lastExpr, maxArg, JIT_RANGE_ITERATOR_START, rangeScanType, outerRow, subQueryIndex); - AddFillKeyPattern(ctx, 0x00, offset, size, JIT_RANGE_ITERATOR_END, rangeScanType, subQueryIndex); - *beginRangeBound = - (indexScan->_last_dim_op1 == JIT_WOC_LESS_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - *endRangeBound = JIT_RANGE_BOUND_INCLUDE; - } else { - // this is a lower bound operator on a descending semi-open scan so we fill the begin key with 0xFF, and - // the end key with the value - AddFillKeyPattern(ctx, 0xFF, offset, size, JIT_RANGE_ITERATOR_START, rangeScanType, subQueryIndex); - buildScanExpression(ctx, lastExpr, maxArg, JIT_RANGE_ITERATOR_END, rangeScanType, outerRow, subQueryIndex); - *beginRangeBound = JIT_RANGE_BOUND_INCLUDE; - *endRangeBound = - (indexScan->_last_dim_op1 == JIT_WOC_GREATER_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - } -} - -static bool buildSemiOpenRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* indexScan, int* maxArg, - JitRangeScanType rangeScanType, JitRangeBoundMode* beginRangeBound, JitRangeBoundMode* endRangeBound, - llvm::Value* outerRow, int subQueryIndex) -{ - // an open range scan starts just like a point scan (with not enough search expressions) and then adds key patterns - // we do not use the last search expression - bool result = buildPointScan(ctx, - &indexScan->_search_exprs, - maxArg, - rangeScanType, - outerRow, - indexScan->_search_exprs._count - 1, - subQueryIndex); - if (result) { - AddCopyKey(ctx, rangeScanType, subQueryIndex); - - // now fill each key with the right pattern for the missing pkey columns in the where clause - bool ascending = (indexScan->_sort_order == JIT_QUERY_SORT_ASCENDING); - - int* index_column_offsets = nullptr; - const uint16_t* key_length = nullptr; - int index_column_count = 0; - if (rangeScanType == JIT_RANGE_SCAN_INNER) { - index_column_offsets = ctx->_inner_table_info.m_indexColumnOffsets; - key_length = ctx->_inner_table_info.m_index->GetLengthKeyFields(); - index_column_count = ctx->_inner_table_info.m_index->GetNumFields(); - } else if (rangeScanType == JIT_RANGE_SCAN_MAIN) { - index_column_offsets = ctx->_table_info.m_indexColumnOffsets; - key_length = ctx->_table_info.m_index->GetLengthKeyFields(); - index_column_count = ctx->_table_info.m_index->GetNumFields(); - } else if (rangeScanType == JIT_RANGE_SCAN_SUB_QUERY) { - index_column_offsets = ctx->m_subQueryTableInfo[subQueryIndex].m_indexColumnOffsets; - key_length = ctx->m_subQueryTableInfo[subQueryIndex].m_index->GetLengthKeyFields(); - index_column_count = ctx->m_subQueryTableInfo[subQueryIndex].m_index->GetNumFields(); - } - - // prepare offset and size for last column in search - int last_dim_column = indexScan->_column_count - 1; - int offset = index_column_offsets[last_dim_column]; - int size = key_length[last_dim_column]; - - // now we fill the last dimension (override extra work of point scan above) - int last_expr_index = indexScan->_search_exprs._count - 1; - JitColumnExpr* last_expr = &indexScan->_search_exprs._exprs[last_expr_index]; - if (ascending) { - BuildAscendingSemiOpenRangeScan(ctx, - indexScan, - maxArg, - rangeScanType, - beginRangeBound, - endRangeBound, - outerRow, - subQueryIndex, - offset, - size, - last_expr); - } else { - BuildDescendingSemiOpenRangeScan(ctx, - indexScan, - maxArg, - rangeScanType, - beginRangeBound, - endRangeBound, - outerRow, - subQueryIndex, - offset, - size, - last_expr); - } - - // now fill the rest as usual - int first_zero_column = indexScan->_column_count; - for (int i = first_zero_column; i < index_column_count; ++i) { - int offset = index_column_offsets[i]; - int size = key_length[i]; - MOT_LOG_DEBUG( - "Filling begin/end iterator pattern for missing pkey fields at offset %d, size %d", offset, size); - AddFillKeyPattern( - ctx, ascending ? 0x00 : 0xFF, offset, size, JIT_RANGE_ITERATOR_START, rangeScanType, subQueryIndex); - AddFillKeyPattern( - ctx, ascending ? 0xFF : 0x00, offset, size, JIT_RANGE_ITERATOR_END, rangeScanType, subQueryIndex); - } - - AddAdjustKey(ctx, - ascending ? 0x00 : 0xFF, - JIT_RANGE_ITERATOR_START, - rangeScanType, // currently this is relevant only for secondary index searches - subQueryIndex); - AddAdjustKey(ctx, - ascending ? 0xFF : 0x00, - JIT_RANGE_ITERATOR_END, - rangeScanType, // currently this is relevant only for secondary index searches - subQueryIndex); - } - return result; -} - -static void BuildAscendingOpenRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* indexScan, int* maxArg, - JitRangeScanType rangeScanType, JitRangeBoundMode* beginRangeBound, JitRangeBoundMode* endRangeBound, - llvm::Value* outerRow, int subQueryIndex, JitWhereOperatorClass beforeLastDimOp, JitWhereOperatorClass lastDimOp, - JitColumnExpr* beforeLastExpr, JitColumnExpr* lastExpr) -{ - if ((beforeLastDimOp == JIT_WOC_LESS_THAN) || (beforeLastDimOp == JIT_WOC_LESS_EQUALS)) { - MOT_ASSERT((lastDimOp == JIT_WOC_GREATER_THAN) || (lastDimOp == JIT_WOC_GREATER_EQUALS)); - // the before-last operator is an upper bound operator on an ascending open scan so we fill the begin - // key with the last value, and the end key with the before-last value - buildScanExpression(ctx, - lastExpr, - maxArg, - JIT_RANGE_ITERATOR_START, - rangeScanType, - outerRow, // lower bound on begin iterator key - subQueryIndex); - buildScanExpression(ctx, - beforeLastExpr, - maxArg, - JIT_RANGE_ITERATOR_END, - rangeScanType, - outerRow, // upper bound on end iterator key - subQueryIndex); - *beginRangeBound = (lastDimOp == JIT_WOC_GREATER_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - *endRangeBound = (beforeLastDimOp == JIT_WOC_LESS_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - } else { - MOT_ASSERT((lastDimOp == JIT_WOC_LESS_THAN) || (lastDimOp == JIT_WOC_LESS_EQUALS)); - // the before-last operator is a lower bound operator on an ascending open scan so we fill the begin key - // with the before-last value, and the end key with the last value - buildScanExpression(ctx, - beforeLastExpr, - maxArg, - JIT_RANGE_ITERATOR_START, - rangeScanType, - outerRow, // lower bound on begin iterator key - subQueryIndex); - buildScanExpression(ctx, - lastExpr, - maxArg, - JIT_RANGE_ITERATOR_END, - rangeScanType, - outerRow, // upper bound on end iterator key - subQueryIndex); - *beginRangeBound = - (beforeLastDimOp == JIT_WOC_GREATER_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - *endRangeBound = (lastDimOp == JIT_WOC_LESS_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - } -} - -static void BuildDescendingOpenRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* indexScan, int* maxArg, - JitRangeScanType rangeScanType, JitRangeBoundMode* beginRangeBound, JitRangeBoundMode* endRangeBound, - llvm::Value* outerRow, int subQueryIndex, JitWhereOperatorClass beforeLastDimOp, JitWhereOperatorClass lastDimOp, - JitColumnExpr* beforeLastExpr, JitColumnExpr* lastExpr) -{ - if ((beforeLastDimOp == JIT_WOC_LESS_THAN) || (beforeLastDimOp == JIT_WOC_LESS_EQUALS)) { - MOT_ASSERT((lastDimOp == JIT_WOC_GREATER_THAN) || (lastDimOp == JIT_WOC_GREATER_EQUALS)); - // the before-last operator is an upper bound operator on an descending open scan so we fill the begin - // key with the last value, and the end key with the before-last value - buildScanExpression(ctx, - beforeLastExpr, - maxArg, - JIT_RANGE_ITERATOR_START, - rangeScanType, - outerRow, // upper bound on begin iterator key - subQueryIndex); - buildScanExpression(ctx, - lastExpr, - maxArg, - JIT_RANGE_ITERATOR_END, - rangeScanType, - outerRow, // lower bound on end iterator key - subQueryIndex); - *beginRangeBound = (beforeLastDimOp == JIT_WOC_LESS_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - *endRangeBound = (lastDimOp == JIT_WOC_GREATER_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - } else { - MOT_ASSERT((lastDimOp == JIT_WOC_LESS_THAN) || (lastDimOp == JIT_WOC_LESS_EQUALS)); - // the before-last operator is a lower bound operator on an descending open scan so we fill the begin - // key with the last value, and the end key with the before-last value - buildScanExpression(ctx, - lastExpr, - maxArg, - JIT_RANGE_ITERATOR_START, - rangeScanType, - outerRow, // upper bound on begin iterator key - subQueryIndex); - buildScanExpression(ctx, - beforeLastExpr, - maxArg, - JIT_RANGE_ITERATOR_END, - rangeScanType, - outerRow, // lower bound on end iterator key - subQueryIndex); - *beginRangeBound = (lastDimOp == JIT_WOC_LESS_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - *endRangeBound = - (beforeLastDimOp == JIT_WOC_GREATER_EQUALS) ? JIT_RANGE_BOUND_INCLUDE : JIT_RANGE_BOUND_EXCLUDE; - } -} - -static bool buildOpenRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* index_scan, int* max_arg, - JitRangeScanType range_scan_type, JitRangeBoundMode* begin_range_bound, JitRangeBoundMode* end_range_bound, - llvm::Value* outer_row, int subQueryIndex) -{ - // an open range scan starts just like a point scan (with not enough search expressions) and then adds key patterns - // we do not use the last two expressions - bool result = buildPointScan(ctx, - &index_scan->_search_exprs, - max_arg, - range_scan_type, - outer_row, - index_scan->_search_exprs._count - 2, - subQueryIndex); - if (result) { - AddCopyKey(ctx, range_scan_type, subQueryIndex); - - // now fill each key with the right pattern for the missing pkey columns in the where clause - bool ascending = (index_scan->_sort_order == JIT_QUERY_SORT_ASCENDING); - - int* index_column_offsets = nullptr; - const uint16_t* key_length = nullptr; - int index_column_count = 0; - if (range_scan_type == JIT_RANGE_SCAN_INNER) { - index_column_offsets = ctx->_inner_table_info.m_indexColumnOffsets; - key_length = ctx->_inner_table_info.m_index->GetLengthKeyFields(); - index_column_count = ctx->_inner_table_info.m_index->GetNumFields(); - } else if (range_scan_type == JIT_RANGE_SCAN_MAIN) { - index_column_offsets = ctx->_table_info.m_indexColumnOffsets; - key_length = ctx->_table_info.m_index->GetLengthKeyFields(); - index_column_count = ctx->_table_info.m_index->GetNumFields(); - } else if (range_scan_type == JIT_RANGE_SCAN_SUB_QUERY) { - index_column_offsets = ctx->m_subQueryTableInfo[subQueryIndex].m_indexColumnOffsets; - key_length = ctx->m_subQueryTableInfo[subQueryIndex].m_index->GetLengthKeyFields(); - index_column_count = ctx->m_subQueryTableInfo[subQueryIndex].m_index->GetNumFields(); - } - - // now we fill the last dimension (override extra work of point scan above) - JitWhereOperatorClass before_last_dim_op = index_scan->_last_dim_op1; // avoid confusion, and give proper names - JitWhereOperatorClass last_dim_op = index_scan->_last_dim_op2; // avoid confusion, and give proper names - int last_expr_index = index_scan->_search_exprs._count - 1; - JitColumnExpr* last_expr = &index_scan->_search_exprs._exprs[last_expr_index]; - JitColumnExpr* before_last_expr = &index_scan->_search_exprs._exprs[last_expr_index - 1]; - if (ascending) { - BuildAscendingOpenRangeScan(ctx, - index_scan, - max_arg, - range_scan_type, - begin_range_bound, - end_range_bound, - outer_row, - subQueryIndex, - before_last_dim_op, - last_dim_op, - before_last_expr, - last_expr); - } else { - BuildDescendingOpenRangeScan(ctx, - index_scan, - max_arg, - range_scan_type, - begin_range_bound, - end_range_bound, - outer_row, - subQueryIndex, - before_last_dim_op, - last_dim_op, - before_last_expr, - last_expr); - } - - // now fill the rest as usual - int first_zero_column = index_scan->_column_count; - for (int i = first_zero_column; i < index_column_count; ++i) { - int offset = index_column_offsets[i]; - int size = key_length[i]; - MOT_LOG_DEBUG( - "Filling begin/end iterator pattern for missing pkey fields at offset %d, size %d", offset, size); - AddFillKeyPattern( - ctx, ascending ? 0x00 : 0xFF, offset, size, JIT_RANGE_ITERATOR_START, range_scan_type, subQueryIndex); - AddFillKeyPattern( - ctx, ascending ? 0xFF : 0x00, offset, size, JIT_RANGE_ITERATOR_END, range_scan_type, subQueryIndex); - } - - AddAdjustKey(ctx, - ascending ? 0x00 : 0xFF, - JIT_RANGE_ITERATOR_START, - range_scan_type, // currently this is relevant only for secondary index searches - subQueryIndex); - AddAdjustKey(ctx, - ascending ? 0xFF : 0x00, - JIT_RANGE_ITERATOR_END, - range_scan_type, // currently this is relevant only for secondary index searches - subQueryIndex); - } - return result; -} - -static bool buildRangeScan(JitLlvmCodeGenContext* ctx, JitIndexScan* index_scan, int* max_arg, - JitRangeScanType range_scan_type, JitRangeBoundMode* begin_range_bound, JitRangeBoundMode* end_range_bound, - llvm::Value* outer_row, int subQueryIndex = -1) -{ - bool result = false; - - // if this is a point scan we generate two identical keys for the iterators - if (index_scan->_scan_type == JIT_INDEX_SCAN_POINT) { - result = - buildPointScan(ctx, &index_scan->_search_exprs, max_arg, range_scan_type, outer_row, -1, subQueryIndex); - if (result) { - AddCopyKey(ctx, range_scan_type, subQueryIndex); - *begin_range_bound = JIT_RANGE_BOUND_INCLUDE; - *end_range_bound = JIT_RANGE_BOUND_INCLUDE; - } - } else if (index_scan->_scan_type == JIT_INDEX_SCAN_CLOSED) { - result = buildClosedRangeScan(ctx, index_scan, max_arg, range_scan_type, outer_row, subQueryIndex); - if (result) { - *begin_range_bound = JIT_RANGE_BOUND_INCLUDE; - *end_range_bound = JIT_RANGE_BOUND_INCLUDE; - } - } else if (index_scan->_scan_type == JIT_INDEX_SCAN_SEMI_OPEN) { - result = buildSemiOpenRangeScan( - ctx, index_scan, max_arg, range_scan_type, begin_range_bound, end_range_bound, outer_row, subQueryIndex); - } else if (index_scan->_scan_type == JIT_INDEX_SCAN_OPEN) { - result = buildOpenRangeScan( - ctx, index_scan, max_arg, range_scan_type, begin_range_bound, end_range_bound, outer_row, subQueryIndex); - } else if (index_scan->_scan_type == JIT_INDEX_SCAN_FULL) { - result = true; // no keys used - *begin_range_bound = JIT_RANGE_BOUND_INCLUDE; - *end_range_bound = JIT_RANGE_BOUND_INCLUDE; - } - return result; -} - -static bool buildPrepareStateScan(JitLlvmCodeGenContext* ctx, JitIndexScan* index_scan, int* max_arg, - JitRangeScanType range_scan_type, llvm::Value* outer_row) -{ - JitRangeBoundMode begin_range_bound = JIT_RANGE_BOUND_NONE; - JitRangeBoundMode end_range_bound = JIT_RANGE_BOUND_NONE; - - // emit code to check if state iterators are null - JIT_IF_BEGIN(state_iterators_exist) - llvm::Value* is_begin_itr_null = AddIsStateIteratorNull(ctx, JIT_RANGE_ITERATOR_START, range_scan_type); - JIT_IF_EVAL(is_begin_itr_null) - // prepare search keys - if (!buildRangeScan(ctx, index_scan, max_arg, range_scan_type, &begin_range_bound, &end_range_bound, outer_row)) { - MOT_LOG_TRACE("Failed to generate jitted code for range select query: unsupported WHERE clause type"); - return false; - } - - // search begin iterator and save it in execution state - IssueDebugLog("Building search iterator from search key, and saving in execution state"); - if (index_scan->_scan_type == JIT_INDEX_SCAN_FULL) { - llvm::Value* itr = buildBeginIterator(ctx, range_scan_type); - AddSetStateIterator(ctx, itr, JIT_RANGE_ITERATOR_START, range_scan_type); - } else { - llvm::Value* itr = buildSearchIterator(ctx, index_scan->_scan_direction, begin_range_bound, range_scan_type); - AddSetStateIterator(ctx, itr, JIT_RANGE_ITERATOR_START, range_scan_type); - - // create end iterator and save it in execution state - IssueDebugLog("Creating end iterator from end search key, and saving in execution state"); - itr = AddCreateEndIterator(ctx, index_scan->_scan_direction, end_range_bound, range_scan_type); - AddSetStateIterator(ctx, itr, JIT_RANGE_ITERATOR_END, range_scan_type); - } - - // initialize state scan variables - if (range_scan_type == JIT_RANGE_SCAN_MAIN) { - AddResetStateLimitCounter(ctx); // in case there is a limit clause - AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); // in case this is a join query - AddSetStateScanEndFlag( - ctx, 0, JIT_RANGE_SCAN_MAIN); // reset state flag (only once, and not repeatedly if row filter failed) - } - JIT_IF_END() - - return true; -} - -static bool buildPrepareStateRow(JitLlvmCodeGenContext* ctx, MOT::AccessType access_mode, JitIndexScan* index_scan, - int* max_arg, JitRangeScanType range_scan_type, llvm::BasicBlock* next_block) -{ - llvm::LLVMContext& context = ctx->_code_gen->context(); - - MOT_LOG_DEBUG("Generating select code for stateful range select"); - llvm::Value* row = nullptr; - - // we start a new block so current block must end with terminator - DEFINE_BLOCK(prepare_state_row_bb, ctx->m_jittedQuery); - MOT_LOG_TRACE("Adding unconditional branch into prepare_state_row_bb from branch %s", - ctx->_builder->GetInsertBlock()->getName().data()); - ctx->_builder->CreateBr(prepare_state_row_bb); - if (MOT_CHECK_LOG_LEVEL(MOT::LogLevel::LL_TRACE)) { - MOT_LOG_BEGIN(MOT::LogLevel::LL_TRACE, "Added unconditional branch result:"); - std::string s; - llvm::raw_string_ostream os(s); - ctx->_builder->GetInsertBlock()->back().print(os); - MOT_LOG_APPEND(MOT::LogLevel::LL_TRACE, "%s", s.c_str()); - MOT_LOG_END(MOT::LogLevel::LL_TRACE); - } - ctx->_builder->SetInsertPoint(prepare_state_row_bb); - - // check if state row is nullptr - JIT_IF_BEGIN(test_state_row) - IssueDebugLog("Checking if state row is nullptr"); - llvm::Value* res = AddIsStateRowNull(ctx, range_scan_type); - JIT_IF_EVAL(res) - IssueDebugLog("State row is nullptr, fetching from state iterators"); - - // check if state scan ended - JIT_IF_BEGIN(test_scan) - IssueDebugLog("Checking if state scan ended"); - llvm::BasicBlock* isStateScanEndBlock = JIT_CURRENT_BLOCK(); // remember current block if filter fails - llvm::Value* res = AddIsStateScanEnd(ctx, index_scan->_scan_direction, range_scan_type); - JIT_IF_EVAL(res) - // fail scan block - IssueDebugLog("Scan ended, raising internal state scan end flag"); - AddSetStateScanEndFlag(ctx, 1, range_scan_type); - JIT_ELSE() - // now get row from iterator - IssueDebugLog("State scan not ended - Retrieving row from iterator"); - row = AddGetRowFromStateIterator(ctx, access_mode, index_scan->_scan_direction, range_scan_type); - - // check if row was found - JIT_IF_BEGIN(test_row_found) - JIT_IF_EVAL_NOT(row) - // row not found branch - IssueDebugLog("Could not retrieve row from state iterator, raising internal state scan end flag"); - AddSetStateScanEndFlag(ctx, 1, range_scan_type); - JIT_ELSE() - // row found, check for additional filters, if not passing filter then go back to execute IsStateScanEnd() - if (!buildFilterRow(ctx, row, &index_scan->_filters, max_arg, isStateScanEndBlock)) { - MOT_LOG_TRACE("Failed to generate jitted code for query: failed to build filter expressions for row"); - return false; - } - // row passed all filters, so save it in state outer row - AddSetStateRow(ctx, row, range_scan_type); - if (range_scan_type == JIT_RANGE_SCAN_MAIN) { - AddSetStateScanEndFlag(ctx, 0, JIT_RANGE_SCAN_INNER); // reset inner scan flag for JOIN queries - } - JIT_IF_END() - JIT_IF_END() - JIT_IF_END() - - // cleanup state iterators if needed and return/jump to next block - JIT_IF_BEGIN(state_scan_ended) - IssueDebugLog("Checking if state scan ended flag was raised"); - llvm::Value* state_scan_end_flag = AddGetStateScanEndFlag(ctx, range_scan_type); - JIT_IF_EVAL(state_scan_end_flag) - IssueDebugLog(" State scan ended flag was raised, cleaning up iterators and reporting to user"); - AddDestroyStateIterators(ctx, range_scan_type); // cleanup - if ((range_scan_type == JIT_RANGE_SCAN_MAIN) || (next_block == nullptr)) { - // either a main scan ended (simple range or outer loop of join), - // or an inner range ended in an outer point join (so next block is null) - // in either case let caller know scan ended and return appropriate value - AddSetScanEnded(ctx, 1); // no outer row, we are definitely done - JIT_RETURN_CONST(MOT::RC_LOCAL_ROW_NOT_FOUND); - } else { - AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); // make sure a new row is fetched in outer loop - ctx->_builder->CreateBr(next_block); // jump back to outer loop - } - JIT_IF_END() - - return true; -} - -llvm::Value* buildPrepareStateScanRow(JitLlvmCodeGenContext* ctx, JitIndexScan* index_scan, - JitRangeScanType range_scan_type, MOT::AccessType access_mode, int* max_arg, llvm::Value* outer_row, - llvm::BasicBlock* next_block, llvm::BasicBlock** loop_block) -{ - // prepare stateful scan if not done so already - if (!buildPrepareStateScan(ctx, index_scan, max_arg, range_scan_type, outer_row)) { - MOT_LOG_TRACE("Failed to generate jitted code for range JOIN query: unsupported %s WHERE clause type", - (range_scan_type == JIT_RANGE_SCAN_MAIN) ? "outer" : "inner"); - return nullptr; - } - - // mark position for later jump - if (loop_block != nullptr) { - *loop_block = llvm::BasicBlock::Create(ctx->_code_gen->context(), "fetch_outer_row_bb", ctx->m_jittedQuery); - ctx->_builder->CreateBr(*loop_block); // end current block - ctx->_builder->SetInsertPoint(*loop_block); // start new block - } - - // fetch row for read - if (!buildPrepareStateRow(ctx, access_mode, index_scan, max_arg, range_scan_type, next_block)) { - MOT_LOG_TRACE("Failed to generate jitted code for range JOIN query: failed to build search outer row block"); - return nullptr; - } - llvm::Value* row = AddGetStateRow(ctx, range_scan_type); - return row; -} - -JitLlvmRuntimeCursor buildRangeCursor(JitLlvmCodeGenContext* ctx, JitIndexScan* indexScan, int* maxArg, - JitRangeScanType rangeScanType, JitIndexScanDirection indexScanDirection, llvm::Value* outerRow, - int subQueryIndex /* = -1 */) -{ - JitLlvmRuntimeCursor result = {nullptr, nullptr}; - JitRangeBoundMode beginRangeBound = JIT_RANGE_BOUND_NONE; - JitRangeBoundMode endRangeBound = JIT_RANGE_BOUND_NONE; - if (!buildRangeScan( - ctx, indexScan, maxArg, rangeScanType, &beginRangeBound, &endRangeBound, outerRow, subQueryIndex)) { - MOT_LOG_TRACE( - "Failed to generate jitted code for aggregate range JOIN query: unsupported %s-loop WHERE clause type", - outerRow ? "inner" : "outer"); - return result; - } - - // build range iterators - result.begin_itr = buildSearchIterator(ctx, indexScanDirection, beginRangeBound, rangeScanType, subQueryIndex); - result.end_itr = - AddCreateEndIterator(ctx, indexScanDirection, endRangeBound, rangeScanType, subQueryIndex); // forward scan - - return result; -} - -bool prepareAggregateAvg(JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate) -{ - // although we already have this information in the aggregate descriptor, we still check again - switch (aggregate->_func_id) { - case INT8AVGFUNCOID: - case NUMERICAVGFUNCOID: - // the current_aggregate is a 2 numeric array - AddPrepareAvgArray(ctx, NUMERICOID, 2); - break; - - case INT4AVGFUNCOID: - case INT2AVGFUNCOID: - case 5537: // int1 avg - // the current_aggregate is a 2 int8 array - AddPrepareAvgArray(ctx, INT8OID, 2); - break; - - case 2104: // float4 - case 2105: // float8 - // the current_aggregate is a 3 float8 array - AddPrepareAvgArray(ctx, FLOAT8OID, 3); - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate AVG() operator function type: %d", aggregate->_func_id); - return false; - } - - return true; -} - -bool prepareAggregateSum(JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate) -{ - switch (aggregate->_func_id) { - case INT8SUMFUNCOID: - // current sum in numeric, and value is int8, both can be null - AddResetAggValue(ctx, NUMERICOID); - break; - - case INT4SUMFUNCOID: - case INT2SUMFUNCOID: - // current aggregate is a int8, and value is int4, both can be null - AddResetAggValue(ctx, INT8OID); - break; - - case 2110: // float4 - // current aggregate is a float4, and value is float4, both can **NOT** be null - AddResetAggValue(ctx, FLOAT4OID); - break; - - case 2111: // float8 - // current aggregate is a float8, and value is float8, both can **NOT** be null - AddResetAggValue(ctx, FLOAT8OID); - break; - - case NUMERICSUMFUNCOID: - AddResetAggValue(ctx, NUMERICOID); - // current aggregate is a numeric, and value is numeric, both can **NOT** be null - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate SUM() operator function type: %d", aggregate->_func_id); - return false; - } - return true; -} - -bool prepareAggregateMaxMin(JitLlvmCodeGenContext* ctx, JitAggregate* aggregate) -{ - AddResetAggMaxMinNull(ctx); - return true; -} - -bool prepareAggregateCount(JitLlvmCodeGenContext* ctx, JitAggregate* aggregate) -{ - switch (aggregate->_func_id) { - case 2147: // int8inc_any - case 2803: // int8inc - // current aggregate is int8, and can **NOT** be null - AddResetAggValue(ctx, INT8OID); - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate COUNT() operator function type: %d", aggregate->_func_id); - return false; - } - return true; -} - -static bool prepareDistinctSet(JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate) -{ - // we need a hash-set according to the aggregated type (preferably but not necessarily linear-probing hash) - // we use an opaque datum type, with a tailor-made hash-function and equals function - AddPrepareDistinctSet(ctx, aggregate->_element_type); - return true; -} - -bool prepareAggregate(JitLlvmCodeGenContext* ctx, JitAggregate* aggregate) -{ - bool result = false; - - switch (aggregate->_aggreaget_op) { - case JIT_AGGREGATE_AVG: - result = prepareAggregateAvg(ctx, aggregate); - break; - - case JIT_AGGREGATE_SUM: - result = prepareAggregateSum(ctx, aggregate); - break; - - case JIT_AGGREGATE_MAX: - case JIT_AGGREGATE_MIN: - result = prepareAggregateMaxMin(ctx, aggregate); - break; - - case JIT_AGGREGATE_COUNT: - result = prepareAggregateCount(ctx, aggregate); - break; - - default: - MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, - "JIT Compile", - "Cannot prepare for aggregation: invalid aggregate operator %d", - (int)aggregate->_aggreaget_op); - break; - } - - if (result) { - if (aggregate->_distinct) { - result = prepareDistinctSet(ctx, aggregate); - } - } - - return result; -} - -static llvm::Value* buildAggregateAvg( - JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate, llvm::Value* current_aggregate, llvm::Value* var_expr) -{ - llvm::Value* aggregate_expr = nullptr; - switch (aggregate->_func_id) { - case INT8AVGFUNCOID: - // the current_aggregate is a 2 numeric array, and the var expression should evaluate to int8 - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int8_avg_accum, 0); - break; - - case INT4AVGFUNCOID: - // the current_aggregate is a 2 int8 array, and the var expression should evaluate to int4 - // we can save a palloc by ensuring "AggCheckCallContext(fcinfo, nullptr)" return true - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int4_avg_accum, 0); - break; - - case INT2AVGFUNCOID: - // the current_aggregate is a 2 int8 array, and the var expression should evaluate to int2 - // we can save a palloc by ensuring "AggCheckCallContext(fcinfo, nullptr)" return true - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int2_avg_accum, 0); - break; - - case 5537: - // the current_aggregate is a 2 int8 array, and the var expression should evaluate to int1 - // we can save a palloc by ensuring "AggCheckCallContext(fcinfo, nullptr)" return true - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int1_avg_accum, 0); - break; - - case 2104: // float4 - // the current_aggregate is a 3 float8 array, and the var expression should evaluate to float4 - // the function computes 3 values: count, sum, square sum - // we can save a palloc by ensuring "AggCheckCallContext(fcinfo, nullptr)" return true - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float4_accum, 0); - break; - - case 2105: // float8 - // the current_aggregate is a 3 float8 array, and the var expression should evaluate to float8 - // the function computes 3 values: count, sum, square sum - // we can save a palloc by ensuring "AggCheckCallContext(fcinfo, nullptr)" return true - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float8_accum, 0); - break; - - case NUMERICAVGFUNCOID: - // the current_aggregate is a 2 numeric array, and the var expression should evaluate to numeric - aggregate_expr = - AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_numeric_avg_accum, 0); - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate AVG() operator function type: %d", aggregate->_func_id); - break; - } - - return aggregate_expr; -} - -static llvm::Value* buildAggregateSum( - JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate, llvm::Value* current_aggregate, llvm::Value* var_expr) -{ - llvm::Value* aggregate_expr = nullptr; - switch (aggregate->_func_id) { - case INT8SUMFUNCOID: - // current sum in numeric, and next value is int8, both can be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int8_sum, 0); - break; - - case INT4SUMFUNCOID: - // current aggregate is a int8, and value is int4, both can be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int4_sum, 0); - break; - - case INT2SUMFUNCOID: - // current aggregate is a int8, and value is int3, both can be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int2_sum, 0); - break; - - case 2110: // float4 - // current aggregate is a float4, and value is float4, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float4pl, 0); - break; - - case 2111: // float8 - // current aggregate is a float4, and value is float4, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float8pl, 0); - break; - - case NUMERICSUMFUNCOID: - // current aggregate is a numeric, and value is numeric, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_numeric_add, 0); - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate SUM() operator function type: %d", aggregate->_func_id); - break; - } - - return aggregate_expr; -} - -static llvm::Value* buildAggregateMax( - JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate, llvm::Value* current_aggregate, llvm::Value* var_expr) -{ - llvm::Value* aggregate_expr = nullptr; - switch (aggregate->_func_id) { - case INT8LARGERFUNCOID: - // current aggregate is a int8, and value is int8, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int8larger, 0); - break; - - case INT4LARGERFUNCOID: - // current aggregate is a int4, and value is int4, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int4larger, 0); - break; - - case INT2LARGERFUNCOID: - // current aggregate is a int2, and value is int2, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int2larger, 0); - break; - - case 5538: - // current aggregate is a int1, and value is int1, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int1larger, 0); - break; - - case 2119: // float4larger - // current aggregate is a float4, and value is float4, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float4larger, 0); - break; - - case 2120: // float8larger - // current aggregate is a float8, and value is float8, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float8larger, 0); - break; - - case NUMERICLARGERFUNCOID: - // current aggregate is a numeric, and value is numeric, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_numeric_larger, 0); - break; - - case 2126: - // current aggregate is a timestamp, and value is timestamp, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_timestamp_larger, 0); - break; - - case 2122: - // current aggregate is a date, and value is date, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_date_larger, 0); - break; - - case 2244: - // current aggregate is a bpchar, and value is bpchar, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_bpchar_larger, 0); - break; - - case 2129: - // current aggregate is a text, and value is text, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_text_larger, 0); - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate MAX() operator function type: %d", aggregate->_func_id); - break; - } - - return aggregate_expr; -} - -static llvm::Value* buildAggregateMin( - JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate, llvm::Value* current_aggregate, llvm::Value* var_expr) -{ - llvm::Value* aggregate_expr = nullptr; - switch (aggregate->_func_id) { - case INT8SMALLERFUNCOID: - // current sum is a int8, and value is int8, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int8smaller, 0); - break; - - case INT4SMALLERFUNCOID: - // current aggregate is a int4, and value is int4, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int4smaller, 0); - break; - - case INT2SMALLERFUNCOID: - // current aggregate is a int2, and value is int2, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_int2smaller, 0); - break; - - case 2135: // float4smaller - // current aggregate is a float4, and value is float4, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float4smaller, 0); - break; - - case 2120: // float8smaller - // current aggregate is a float8, and value is float8, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_float8smaller, 0); - break; - - case NUMERICSMALLERFUNCOID: - // current aggregate is a numeric, and value is numeric, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_numeric_smaller, 0); - break; - - case 2142: - // current aggregate is a timestamp, and value is timestamp, both can **NOT** be null - aggregate_expr = - AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_timestamp_smaller, 0); - break; - - case 2138: - // current aggregate is a date, and value is date, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_date_smaller, 0); - break; - - case 2245: - // current aggregate is a bpchar, and value is bpchar, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_bpchar_smaller, 0); - break; - - case 2145: - // current aggregate is a text, and value is text, both can **NOT** be null - aggregate_expr = AddExecBinaryOperator(ctx, current_aggregate, var_expr, ctx->_builtin_text_smaller, 0); - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate MIN() operator function type: %d", aggregate->_func_id); - break; - } - - return aggregate_expr; -} - -static llvm::Value* buildAggregateCount( - JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate, llvm::Value* count_aggregate) -{ - llvm::Value* aggregate_expr = nullptr; - switch (aggregate->_func_id) { - case 2147: // int8inc_any - // current aggregate is int8, and can **NOT** be null - aggregate_expr = AddExecUnaryOperator(ctx, count_aggregate, ctx->_builtin_int8inc, 0); - break; - - case 2803: // int8inc - // current aggregate is int8, and can **NOT** be null - aggregate_expr = AddExecUnaryOperator(ctx, count_aggregate, ctx->_builtin_int8inc, 0); - break; - - default: - MOT_LOG_TRACE("Unsupported aggregate COUNT() operator function type: %d", aggregate->_func_id); - break; - } - - return aggregate_expr; -} - -static bool buildAggregateMaxMin(JitLlvmCodeGenContext* ctx, JitAggregate* aggregate, llvm::Value* var_expr) -{ - bool result = false; - llvm::Value* aggregateExpr = nullptr; - - // we first check if the min/max value is null and if so just store the column value - JIT_IF_BEGIN(test_max_min_value_null) - llvm::Value* res = AddGetAggMaxMinIsNull(ctx); - JIT_IF_EVAL(res) - AddSetAggValue(ctx, var_expr); - AddSetAggMaxMinNotNull(ctx); - JIT_ELSE() - // get the aggregated value and call the operator - llvm::Value* current_aggregate = AddGetAggValue(ctx); - switch (aggregate->_aggreaget_op) { - case JIT_AGGREGATE_MAX: - aggregateExpr = buildAggregateMax(ctx, aggregate, current_aggregate, var_expr); - break; - - case JIT_AGGREGATE_MIN: - aggregateExpr = buildAggregateMin(ctx, aggregate, current_aggregate, var_expr); - break; - - default: - MOT_REPORT_ERROR( - MOT_ERROR_INTERNAL, "JIT Compile", "Invalid aggregate operator %d", (int)aggregate->_aggreaget_op); - break; - } - JIT_IF_END() - - if (aggregateExpr != nullptr) { - AddSetAggValue(ctx, aggregateExpr); - result = true; - } - - return result; -} - -static bool buildAggregateTuple(JitLlvmCodeGenContext* ctx, JitAggregate* aggregate, llvm::Value* var_expr) -{ - bool result = false; - - if ((aggregate->_aggreaget_op == JIT_AGGREGATE_MAX) || (aggregate->_aggreaget_op == JIT_AGGREGATE_MIN)) { - result = buildAggregateMaxMin(ctx, aggregate, var_expr); - } else { - // get the aggregated value - llvm::Value* current_aggregate = nullptr; - if (aggregate->_aggreaget_op == JIT_AGGREGATE_AVG) { - current_aggregate = AddLoadAvgArray(ctx); - } else { - current_aggregate = AddGetAggValue(ctx); - } - - // the operators below take care of null inputs - llvm::Value* aggregate_expr = nullptr; - switch (aggregate->_aggreaget_op) { - case JIT_AGGREGATE_AVG: - aggregate_expr = buildAggregateAvg(ctx, aggregate, current_aggregate, var_expr); - break; - - case JIT_AGGREGATE_SUM: - aggregate_expr = buildAggregateSum(ctx, aggregate, current_aggregate, var_expr); - break; - - case JIT_AGGREGATE_COUNT: - aggregate_expr = buildAggregateCount(ctx, aggregate, current_aggregate); - break; - default: - MOT_REPORT_ERROR( - MOT_ERROR_INTERNAL, "JIT Compile", "Invalid aggregate operator %d", (int)aggregate->_aggreaget_op); break; } - // write back the sum to the aggregated value/array - if (aggregate_expr != nullptr) { - if (aggregate->_aggreaget_op == JIT_AGGREGATE_AVG) { - AddSaveAvgArray(ctx, aggregate_expr); - } else { - AddSetAggValue(ctx, aggregate_expr); + if (m_preAbort) { + if (!QuickHeaderValidation(ac)) { + if (MOTEngine::GetInstance()->IsRecovering() && ResolveRecoveryOccConflict(txMan, ac) == RC_OK) { + (void)++itr; + continue; + } + return false; } - result = true; - } else { - MOT_LOG_TRACE("Failed to generate aggregate AVG/SUM/COUNT code"); } - } - - return result; -} - -bool buildAggregateRow( - JitLlvmCodeGenContext* ctx, JitAggregate* aggregate, llvm::Value* row, llvm::BasicBlock* next_block) -{ - bool result = false; - - // extract the aggregated column - llvm::Value* table = (aggregate->_table == ctx->_table_info.m_table) ? ctx->table_value : ctx->inner_table_value; - llvm::Value* expr = AddReadDatumColumn(ctx, table, row, aggregate->_table_column_id, 0); - - // we first check if we have DISTINCT modifier - if (aggregate->_distinct) { - // check row is distinct in state set (managed on the current jit context) - // emit code to convert column to primitive data type, add value to distinct set and verify - // if value is not distinct then do not use this row in aggregation - JIT_IF_BEGIN(test_distinct_value) - llvm::Value* res = AddInsertDistinctItem(ctx, aggregate->_element_type, expr); - JIT_IF_EVAL_NOT(res) - IssueDebugLog("Value is not distinct, skipping aggregated row"); - ctx->_builder->CreateBr(next_block); - JIT_IF_END() - } - - // aggregate row - IssueDebugLog("Aggregating row into inner value"); - result = buildAggregateTuple(ctx, aggregate, expr); - if (result) { - // update number of rows processes - buildIncrementRowsProcessed(ctx); - } - - return result; -} - -void buildAggregateResult(JitLlvmCodeGenContext* ctx, const JitAggregate* aggregate) -{ - // in case of average we compute it when aggregate loop is done - if (aggregate->_aggreaget_op == JIT_AGGREGATE_AVG) { - llvm::Value* avg_value = AddComputeAvgFromArray( - ctx, aggregate->_avg_element_type); // we infer this during agg op analysis, but don't save it... - AddWriteTupleDatum(ctx, 0, avg_value); // we alway aggregate to slot tuple 0 - } else { - llvm::Value* count_value = AddGetAggValue(ctx); - AddWriteTupleDatum(ctx, 0, count_value); // we alway aggregate to slot tuple 0 - } - - // we take the opportunity to cleanup as well - if (aggregate->_distinct) { - AddDestroyDistinctSet(ctx, aggregate->_element_type); - } -} - -void buildCheckLimit(JitLlvmCodeGenContext* ctx, int limit_count) -{ - // if a limit clause exists, then increment limit counter and check if reached limit - if (limit_count > 0) { - AddIncrementStateLimitCounter(ctx); - JIT_IF_BEGIN(limit_count_reached) - llvm::Value* current_limit_count = AddGetStateLimitCounter(ctx); - llvm::Value* limit_count_inst = JIT_CONST(limit_count); - JIT_IF_EVAL_CMP(current_limit_count, limit_count_inst, JIT_ICMP_EQ); - IssueDebugLog("Reached limit specified in limit clause, raising internal state scan end flag"); - // cleanup and signal scan ended (it is safe to cleanup even if not initialized, so we cleanup everything) - AddDestroyStateIterators(ctx, JIT_RANGE_SCAN_MAIN); - AddDestroyStateIterators(ctx, JIT_RANGE_SCAN_INNER); - AddSetScanEnded(ctx, 1); - JIT_IF_END() - } -} - -bool selectJoinRows( - JitLlvmCodeGenContext* ctx, llvm::Value* outer_row_copy, llvm::Value* inner_row, JitJoinPlan* plan, int* max_arg) -{ - // select inner and outer row expressions into result tuple (no aggregate because aggregate is not stateful) - IssueDebugLog("Retrieved row from state iterator, beginning to select columns into result tuple"); - if (!selectRowColumns(ctx, outer_row_copy, &plan->_select_exprs, max_arg, JIT_RANGE_SCAN_MAIN)) { - MOT_LOG_TRACE( - "Failed to generate jitted code for Inner Point JOIN query: failed to select outer row expressions"); - return false; - } - if (!selectRowColumns(ctx, inner_row, &plan->_select_exprs, max_arg, JIT_RANGE_SCAN_INNER)) { - MOT_LOG_TRACE( - "Failed to generate jitted code for Inner Point JOIN query: failed to select outer row expressions"); - return false; + (void)++itr; } return true; } -} // namespace JitExec + +bool OccTransactionManager::QuickVersionCheck(const Access* access) +{ + if (access->m_params.IsPrimarySentinel()) { + // For Upgrade IOD - Verify the sentinel snapshot is still valid + // Check if the key is still visible + if (access->m_snapshot <= access->m_origSentinel->GetStartCSN()) { + return false; + } + MOT_ASSERT(access->m_csn == access->m_globalRow->GetCommitSequenceNumber()); + return (access->m_csn == access->m_origSentinel->GetData()->GetCommitSequenceNumber()); + } else { + if (access->m_params.IsSecondaryUniqueSentinel()) { + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (node->GetEndCSN() != Sentinel::SENTINEL_INIT_CSN) { + return false; + } + return (access->m_secondaryUniqueNode == node); + } else { + MOT_ASSERT(access->GetType() == AccessType::DEL); + // Check that the Sentinel is not deleted! + return (static_cast(access->GetSentinel())->GetEndCSN() == Sentinel::SENTINEL_INIT_CSN); + } + } +} + +bool OccTransactionManager::QuickInsertCheck(const Access* access) +{ + // Lets verify the inserts + Sentinel* sent = access->m_origSentinel; + if (access->m_params.IsUpgradeInsert() == false) { + // if the sent is committed we abort! + if (sent->IsCommited()) { + return false; + } + } else { + if (access->m_params.IsPrimarySentinel()) { + // For Upgrade IOD - Verify the sentinel snapshot is still valid + if (access->m_params.IsInsertOnDeletedRow()) { + if (access->m_origSentinel->GetData()->IsRowDeleted() == false) { + return false; + } + } + return (access->m_csn == access->m_origSentinel->GetData()->GetCommitSequenceNumber()); + } else { + if (access->m_params.IsSecondaryUniqueSentinel()) { + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (access->m_params.IsInsertOnDeletedRow()) { + // Check if node is deleted + if (node->GetEndCSN() == Sentinel::SENTINEL_INIT_CSN) { + return false; + } + } else { + // Check if node is committed + if (node->GetEndCSN() != Sentinel::SENTINEL_INIT_CSN) { + return false; + } + } + return (access->m_secondaryUniqueNode == node); + } else { + MOT_ASSERT(access->GetType() == AccessType::INS); + MOT_ASSERT(access->m_params.IsIndexUpdate() == true); + if (static_cast(access->GetSentinel())->GetEndCSN() == + Sentinel::SENTINEL_INIT_CSN) { + return false; + } + return (static_cast(access->GetSentinel())->GetStartCSN() == access->m_csn); + } + } + } + + return true; +} + +bool OccTransactionManager::QuickHeaderValidation(const Access* access) +{ + if (access->m_type != INS) { + // For WR/DEL/RD_FOR_UPDATE lets verify CSN + return QuickVersionCheck(access); + } else { + return QuickInsertCheck(access); + } +} + +bool OccTransactionManager::ValidateWriteSet(TxnManager* txMan) +{ + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + for (const auto& raPair : orderedSet) { + const Access* ac = raPair.second; + if (!QuickHeaderValidation(ac)) { + return false; + } + } + return true; +} + +RC OccTransactionManager::LockHeaders(TxnManager* txMan, uint32_t& numSentinelsLock) +{ + RC rc = RC_OK; + uint64_t sleepTime = 1; + uint64_t thdId = txMan->GetThdId(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + numSentinelsLock = 0; + if (m_validationNoWait) { + while (numSentinelsLock != m_writeSetSize) { + for (const auto& raPair : orderedSet) { + const Access* ac = raPair.second; + Sentinel* sent = ac->m_origSentinel; + if (!sent->TryLock(thdId)) { + break; + } + numSentinelsLock++; + // New insert row is already committed! + // Check if row has changed in sentinel + if (!QuickHeaderValidation(ac)) { + rc = RC_ABORT; + goto final; + } + } + + if (numSentinelsLock != m_writeSetSize) { + ReleaseHeaderLocks(txMan, numSentinelsLock); + numSentinelsLock = 0; + if (m_preAbort) { + for (const auto& acPair : orderedSet) { + const Access* ac = acPair.second; + if (!QuickHeaderValidation(ac)) { + return RC_ABORT; + } + } + } + if (!MOTEngine::GetInstance()->IsRecovering()) { + if (sleepTime > LOCK_TIME_OUT) { + return RC_ABORT; + } else { + if (!IsHighContention()) { + CpuCyclesLevelTime::Sleep(5); + } else { + (void)usleep(m_dynamicSleep); + } + sleepTime = sleepTime << 1; + } + } else { + (void)usleep(1000); + } + } + } + } else { + for (const auto& raPair : orderedSet) { + const Access* ac = raPair.second; + Sentinel* sent = ac->m_origSentinel; + sent->Lock(thdId); + numSentinelsLock++; + // New insert row is already committed! + // Check if row has changed in sentinel + if (!QuickHeaderValidation(ac)) { + rc = RC_ABORT; + goto final; + } + } + } +final: + return rc; +} + +bool OccTransactionManager::PreAllocStableRow(TxnManager* txMan) +{ + if (GetGlobalConfiguration().m_enableCheckpoint) { + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD || (access->m_type == INS && access->m_params.IsUpgradeInsert() == false)) { + continue; + } + if (access->m_params.IsPrimarySentinel()) { + if (!GetCheckpointManager()->PreAllocStableRow(txMan, access->GetRowFromHeader(), access->m_type)) { + GetCheckpointManager()->FreePreAllocStableRows(txMan); + return false; + } + } + } + } + return true; +} + +bool OccTransactionManager::ReserveGcMemory(TxnManager* txMan, const GcMaintenanceInfo& gcMemoryReserve) +{ + bool res = true; + GcManager* gc_manager = txMan->GetGcSession(); + MOT_ASSERT(gc_manager != nullptr); + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::DELETE_QUEUE, gcMemoryReserve.m_delete_queue); + if (!res) { + return false; + } + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::VERSION_QUEUE, gcMemoryReserve.m_version_queue); + if (!res) { + return false; + } + res = + gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::UPDATE_COLUMN_QUEUE, gcMemoryReserve.m_update_column_queue); + if (!res) { + return false; + } + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::GENERIC_QUEUE, gcMemoryReserve.m_generic_queue, true); + if (!res) { + return false; + } + + return res; +} +/* function name:ValidateOcc + function purpose:Verify that all operations done by the transaction are legal before the transaction is committed, + that is, they do not conflict with other concurrent transactions + input:transaction manager + output:Validation results(enum variable). + note:none + annotator:liushifa + annotate time:2023/08/25 10:59:07 + contact:3325287047@qq.com +*/ +RC OccTransactionManager::ValidateOcc(TxnManager* txMan) +{ // Initialize the number of locked sentinels to 0 + uint32_t numSentinelLock = 0; + m_rowsLocked = false; + // Get the access manager for the transaction + TxnAccess* txnAccess = txMan->m_accessMgr; + RC rc = RC_OK; + const uint32_t rowCount = txnAccess->Size(); + + m_writeSetSize = 0; + m_insertSetSize = 0; + m_txnCounter++; +// If no rows are accessed, the transaction is read-only and returns success directly + if (rowCount == 0) { + // READONLY + return rc; + } +// Initialize garbage collection maintenance information + GcMaintenanceInfo gcMemoryReserve{}; + + MOT_ASSERT(rowCount == txnAccess->GetOrderedRowSet().size()); + + do { + /* 1.Perform pre-abort check and pre-processing */ + if (!PreAbortCheck(txMan, gcMemoryReserve)) { + rc = RC_ABORT; + break; + } + // try to lock all headers + rc = LockHeaders(txMan, numSentinelLock); + if (rc != RC_OK) { + break; + } + // verify write set + if (!ValidateWriteSet(txMan)) { + rc = RC_ABORT; + break; + } + + // Pre-allocate stable row according to the checkpoint state. + if (!PreAllocStableRow(txMan)) { + rc = RC_MEMORY_ALLOCATION_ERROR; + break; + } + // Reserve memory for possible garbage collection operations + if (!ReserveGcMemory(txMan, gcMemoryReserve)) { + rc = RC_MEMORY_ALLOCATION_ERROR; + break; + } + } while (0); + // If all the above steps are successful + if (likely(rc == RC_OK)) { + MOT_ASSERT(numSentinelLock == m_writeSetSize); + // mark the row as locked + m_rowsLocked = true; + } else { + // Release all header locks + ReleaseHeaderLocks(txMan, numSentinelLock); + if (likely(rc == RC_ABORT)) { + m_abortsCounter++; + } + } + + return rc; +} + +RC OccTransactionManager::ResolveRecoveryOccConflict(TxnManager* txMan, Access* access) +{ + Row* row = nullptr; + RC rc = RC_ABORT; + uint64_t endCSN = static_cast(-1); + MOT_ASSERT(access->m_type == INS); + switch (access->m_origSentinel->GetIndexOrder()) { + case IndexOrder::INDEX_ORDER_PRIMARY: + // Check what is the current row the sentinel is pointing + row = access->m_origSentinel->GetData(); + if (row) { + // Row must be deleted with Smaller CSN + if (row->IsRowDeleted() == false) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + MOT_ASSERT(access->m_origSentinel->GetData()->GetCommitSequenceNumber() > access->m_csn); + // Reset the global version and set to insert on delete + access->m_params.SetUpgradeInsert(); + access->m_params.SetInsertOnDeletedRow(); + access->m_globalRow = row; + access->m_csn = row->GetCommitSequenceNumber(); + access->m_snapshot = static_cast(-1); + rc = RC_OK; + } else { + MOT_ASSERT(false); + return RC_ABORT; + } + break; + case IndexOrder::INDEX_ORDER_SECONDARY: + endCSN = static_cast(access->m_origSentinel)->GetEndCSN(); + if (txMan->GetCommitSequenceNumber() <= endCSN) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + if (endCSN == Sentinel::SENTINEL_INIT_CSN) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + access->m_params.SetUpgradeInsert(); + access->m_params.SetInsertOnDeletedRow(); + access->m_csn = static_cast(access->m_origSentinel)->GetStartCSN(); + rc = RC_OK; + break; + case IndexOrder::INDEX_ORDER_SECONDARY_UNIQUE: + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (node != nullptr) { + if (txMan->GetCommitSequenceNumber() <= node->GetEndCSN()) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + } + // Reset Visible node + access->m_params.SetUpgradeInsert(); + if (node->GetEndCSN() < Sentinel::SENTINEL_INIT_CSN) { + access->m_params.SetInsertOnDeletedRow(); + } else { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + access->m_secondaryUniqueNode = node; + rc = RC_OK; + break; + } + return rc; +} +void OccTransactionManager::WriteChanges(TxnManager* txMan) +{ + if (m_writeSetSize == 0 && m_insertSetSize == 0) { + return; + } + + MOTConfiguration& cfg = GetGlobalConfiguration(); + uint64_t commit_csn = txMan->GetCommitSequenceNumber(); + uint64_t transaction_id = txMan->GetInternalTransactionId(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + + // Stable rows for checkpoint needs to be created (copied from original row) before modifying the global rows. + if (cfg.m_enableCheckpoint) { + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD) { + continue; + } + if (access->m_params.IsPrimarySentinel()) { + // Pass the actual global row (access->GetRowFromHeader()), so that the stable row will have the + // same CSN, rowid, etc as the original row before the modifications are applied. + GetCheckpointManager()->ApplyWrite(txMan, access->GetRowFromHeader(), access); + } + } + } + + // Update CSN with all relevant information on global rows + // For deletes invalidate sentinels - rows still locked! + for (const auto& raPair : orderedSet) { + Access* access = raPair.second; + access->WriteGlobalChanges(commit_csn, transaction_id); + } + + WriteSentinelChanges(txMan); + + // For Recovery operation:Update transactionID + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD) { + continue; + } + if (access->m_params.IsPrimarySentinel()) { + static_cast(access->m_origSentinel)->SetTransactionId(transaction_id); + } + } + + m_isTransactionCommited = true; +} + +void OccTransactionManager::WriteSentinelChanges(TxnManager* txMan) +{ + uint64_t commit_csn = txMan->GetCommitSequenceNumber(); + S_SentinelNodePool* sentinelObjectPool = txMan->m_accessMgr->GetSentinelObjectPool(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + + if (m_insertSetSize > 0) { + for (const auto& raPair : orderedSet) { + Access* access = raPair.second; + if (access->m_type != INS) { + continue; + } + MOT_ASSERT(access->m_origSentinel->IsLocked() == true); + if (access->m_params.IsUpgradeInsert() == false) { + if (access->m_params.IsPrimarySentinel()) { + MOT_ASSERT(access->m_origSentinel->IsDirty() == true); + MOT_ASSERT(access->m_origSentinel->IsLocked() == true); + // Connect row and sentinel, row is set to absent and locked + access->m_origSentinel->SetStartCSN(commit_csn); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + // Current state: row is set to absent,sentinel is locked and not dirty + // Readers will not see the row + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + access->GetTxnRow()->GetTable()->UpdateRowCount(1); + } else { + if (access->m_params.IsSecondaryUniqueSentinel() == false) { + // We only set the in the secondary sentinel! + access->m_origSentinel->SetStartCSN(commit_csn); + access->m_origSentinel->SetEndCSN(Sentinel::SENTINEL_INIT_CSN); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()->GetPrimarySentinel()); + } else { + access->m_origSentinel->SetStartCSN(commit_csn); + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + access->m_origSentinel->SetNextPtr(node); + } + } + // Unset Dirty - still locked! + access->m_origSentinel->UnSetDirty(); + } else { + if (access->m_params.IsInsertOnDeletedRow()) { + if (access->m_params.IsPrimarySentinel()) { + access->GetTxnRow()->GetTable()->UpdateRowCount(1); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + } else if (access->m_params.IsUniqueIndex() == true) { + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + auto oldNode = static_cast(access->m_origSentinel)->GetTopNode(); + node->SetNextVersion(oldNode); + access->m_origSentinel->SetNextPtr(node); + } else { + MOT_ASSERT(access->m_params.IsIndexUpdate()); + // Revalidate End CSN + static_cast(access->m_origSentinel)->SetEndCSN(Sentinel::SENTINEL_INIT_CSN); + } + } else { + // We delete row internally and insert a new row + if (access->m_params.IsPrimarySentinel()) { + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + } else { + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + auto oldNode = static_cast(access->m_origSentinel)->GetTopNode(); + oldNode->SetEndCSN(commit_csn); + node->SetNextVersion(oldNode); + access->m_origSentinel->SetNextPtr(node); + } + } + } + } + } +} +/* function name:ReleaseHeaderLocks + function purpose:Releases any row header locks acquired by the transaction during its execution. + input:transaction managers and number of locks + output:none + note:none + annotator:liushifa + annotate time:2023/08/25 11:19:14 + contact:3325287047@qq.com +*/ +void OccTransactionManager::ReleaseHeaderLocks(TxnManager* txMan, uint32_t numOfLocks) +{ + if (numOfLocks == 0) { + return; + } + + TxnAccess* tx = txMan->m_accessMgr; + TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); + // use local counter to optimize + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD) { + continue; + } else { + numOfLocks--; + access->m_origSentinel->Unlock(); + } + if (!numOfLocks) { + break; + } + } +} + +void OccTransactionManager::CleanUp() +{ + m_writeSetSize = 0; + m_insertSetSize = 0; + m_isTransactionCommited = false; +} +} // namespace MOT diff --git a/src/gausskernel/storage/mot/jit_exec/jit_tvm_query_codegen.cpp b/src/gausskernel/storage/mot/jit_exec/jit_tvm_query_codegen.cpp index 974cab4c5..47ba67c16 100644 --- a/src/gausskernel/storage/mot/jit_exec/jit_tvm_query_codegen.cpp +++ b/src/gausskernel/storage/mot/jit_exec/jit_tvm_query_codegen.cpp @@ -13,1772 +13,620 @@ * See the Mulan PSL v2 for more details. * ------------------------------------------------------------------------- * - * jit_tvm_query_codegen.cpp - * TVM-jitted code generation. + * occ_transaction_manager.cpp + * Optimistic Concurrency Control (OCC) implementation * * IDENTIFICATION - * src/gausskernel/storage/mot/jit_exec/jit_tvm_query_codegen.cpp + * src/gausskernel/storage/mot/core/concurrency_control/occ_transaction_manager.h * * ------------------------------------------------------------------------- */ -/* - * ATTENTION: Be sure to include jit_tvm_query.h before anything else because of libintl.h - * See jit_tvm_query.h for more details. - */ -#include "jit_tvm_query.h" -#include "jit_tvm_query_codegen.h" -#include "jit_tvm_funcs.h" -#include "jit_tvm_blocks.h" -#include "storage/mot/jit_exec.h" -#include "jit_tvm_util.h" -#include "jit_util.h" +#include "occ_transaction_manager.h" +#include "utilities.h" +#include "cycles.h" +#include "mot_engine.h" +#include "row.h" +#include "txn.h" +#include "txn_access.h" +#include "checkpoint_manager.h" +#include "mm_session_api.h" +#include "mot_error.h" +#include -using namespace tvm; +namespace MOT { +DECLARE_LOGGER(OccTransactionManager, ConcurrenyControl); -namespace JitExec { -DECLARE_LOGGER(JitTvmQueryCodegen, JitExec) +OccTransactionManager::OccTransactionManager() + : m_txnCounter(0), + m_abortsCounter(0), + m_writeSetSize(0), + m_insertSetSize(0), + m_dynamicSleep(100), + m_rowsLocked(false), + m_preAbort(true), + m_validationNoWait(true), + m_isTransactionCommited(false) +{} -static void DestroyCodeGenContext(JitTvmCodeGenContext* ctx); +OccTransactionManager::~OccTransactionManager() +{} -/** @brief Initializes a context for compilation. */ -static bool InitCodeGenContext(JitTvmCodeGenContext* ctx, Builder* builder, MOT::Table* table, MOT::Index* index, - MOT::Table* inner_table = nullptr, MOT::Index* inner_index = nullptr) +bool OccTransactionManager::PreAbortCheck(TxnManager* txMan, GcMaintenanceInfo& gcMemoryReserve) { - errno_t erc = memset_s(ctx, sizeof(JitTvmCodeGenContext), 0, sizeof(JitTvmCodeGenContext)); - securec_check(erc, "\0", "\0"); - ctx->_builder = builder; - if (!InitTableInfo(&ctx->_table_info, table, index)) { - MOT_REPORT_ERROR( - MOT_ERROR_OOM, "JIT Compile", "Failed to initialize table information for code-generation context"); - return false; - } - if (inner_table && !InitTableInfo(&ctx->m_innerTable_info, inner_table, inner_index)) { - DestroyTableInfo(&ctx->_table_info); - MOT_REPORT_ERROR(MOT_ERROR_OOM, - "JIT Compile", - "Failed to initialize inner-scan table information for code-generation context"); - return false; - } + TxnAccess* tx = txMan->m_accessMgr; + TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); + auto itr = orderedSet.begin(); + while (itr != orderedSet.end()) { + Access* ac = (*itr).second; + switch (ac->m_type) { + case WR: + m_writeSetSize++; + gcMemoryReserve.m_version_queue++; + break; + case DEL: + m_writeSetSize++; + if (ac->m_params.IsPrimarySentinel()) { + gcMemoryReserve.m_delete_queue++; + } else { + if (ac->m_params.IsIndexUpdate()) { + gcMemoryReserve.m_update_column_queue++; + } + } + gcMemoryReserve.m_generic_queue++; + break; + case INS: + m_insertSetSize++; + m_writeSetSize++; + if (ac->m_params.IsUpgradeInsert()) { + gcMemoryReserve.m_version_queue++; + } + break; + case RD_FOR_UPDATE: + case RD: + itr = orderedSet.erase(itr); + txMan->m_accessMgr->PubReleaseAccess(ac); + continue; + default: + break; + } - ctx->m_constCount = 0; - size_t allocSize = sizeof(Const) * MOT_JIT_MAX_CONST; - ctx->m_constValues = (Const*)MOT::MemSessionAlloc(allocSize); - if (ctx->m_constValues == nullptr) { - MOT_REPORT_ERROR(MOT_ERROR_OOM, - "JIT Compile", - "Failed to allocate %u bytes for constant array in code-generation context", - allocSize); - DestroyCodeGenContext(ctx); - return false; + if (m_preAbort) { + if (!QuickHeaderValidation(ac)) { + if (MOTEngine::GetInstance()->IsRecovering() && ResolveRecoveryOccConflict(txMan, ac) == RC_OK) { + (void)++itr; + continue; + } + return false; + } + } + (void)++itr; + } + return true; +} + +bool OccTransactionManager::QuickVersionCheck(const Access* access) +{ + if (access->m_params.IsPrimarySentinel()) { + // For Upgrade IOD - Verify the sentinel snapshot is still valid + // Check if the key is still visible + if (access->m_snapshot <= access->m_origSentinel->GetStartCSN()) { + return false; + } + MOT_ASSERT(access->m_csn == access->m_globalRow->GetCommitSequenceNumber()); + return (access->m_csn == access->m_origSentinel->GetData()->GetCommitSequenceNumber()); + } else { + if (access->m_params.IsSecondaryUniqueSentinel()) { + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (node->GetEndCSN() != Sentinel::SENTINEL_INIT_CSN) { + return false; + } + return (access->m_secondaryUniqueNode == node); + } else { + MOT_ASSERT(access->GetType() == AccessType::DEL); + // Check that the Sentinel is not deleted! + return (static_cast(access->GetSentinel())->GetEndCSN() == Sentinel::SENTINEL_INIT_CSN); + } + } +} + +bool OccTransactionManager::QuickInsertCheck(const Access* access) +{ + // Lets verify the inserts + Sentinel* sent = access->m_origSentinel; + if (access->m_params.IsUpgradeInsert() == false) { + // if the sent is committed we abort! + if (sent->IsCommited()) { + return false; + } + } else { + if (access->m_params.IsPrimarySentinel()) { + // For Upgrade IOD - Verify the sentinel snapshot is still valid + if (access->m_params.IsInsertOnDeletedRow()) { + if (access->m_origSentinel->GetData()->IsRowDeleted() == false) { + return false; + } + } + return (access->m_csn == access->m_origSentinel->GetData()->GetCommitSequenceNumber()); + } else { + if (access->m_params.IsSecondaryUniqueSentinel()) { + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (access->m_params.IsInsertOnDeletedRow()) { + // Check if node is deleted + if (node->GetEndCSN() == Sentinel::SENTINEL_INIT_CSN) { + return false; + } + } else { + // Check if node is committed + if (node->GetEndCSN() != Sentinel::SENTINEL_INIT_CSN) { + return false; + } + } + return (access->m_secondaryUniqueNode == node); + } else { + MOT_ASSERT(access->GetType() == AccessType::INS); + MOT_ASSERT(access->m_params.IsIndexUpdate() == true); + if (static_cast(access->GetSentinel())->GetEndCSN() == + Sentinel::SENTINEL_INIT_CSN) { + return false; + } + return (static_cast(access->GetSentinel())->GetStartCSN() == access->m_csn); + } + } } return true; } -/** @brief Initializes a context for compilation. */ -static bool InitCompoundCodeGenContext( - JitTvmCodeGenContext* ctx, Builder* builder, MOT::Table* table, MOT::Index* index, JitCompoundPlan* plan) +bool OccTransactionManager::QuickHeaderValidation(const Access* access) { - // initialize outer query table info - errno_t erc = memset_s(ctx, sizeof(JitTvmCodeGenContext), 0, sizeof(JitTvmCodeGenContext)); - securec_check(erc, "\0", "\0"); - ctx->_builder = builder; - if (!InitTableInfo(&ctx->_table_info, table, index)) { - MOT_REPORT_ERROR( - MOT_ERROR_OOM, "JIT Compile", "Failed to initialize table information for code-generation context"); - return false; - } - ctx->m_subQueryCount = plan->_sub_query_count; - ctx->m_subQueryTableInfo = (TableInfo*)MOT::MemSessionAlloc(sizeof(TableInfo) * ctx->m_subQueryCount); - if (ctx->m_subQueryTableInfo == nullptr) { - MOT_REPORT_ERROR( - MOT_ERROR_OOM, "JIT Compile", "Failed to initialize table information for code-generation context"); - DestroyTableInfo(&ctx->_table_info); - return false; + if (access->m_type != INS) { + // For WR/DEL/RD_FOR_UPDATE lets verify CSN + return QuickVersionCheck(access); + } else { + return QuickInsertCheck(access); } +} - // initialize sub-query table info - bool result = true; - for (uint32_t i = 0; i < ctx->m_subQueryCount; ++i) { - JitPlan* subPlan = plan->_sub_query_plans[i]; - MOT::Table* subTable = nullptr; - MOT::Index* subIndex = nullptr; - if (subPlan->_plan_type == JIT_PLAN_POINT_QUERY) { - subTable = ((JitSelectPlan*)subPlan)->_query._table; - subIndex = subTable->GetPrimaryIndex(); - } else if (subPlan->_plan_type == JIT_PLAN_RANGE_SCAN) { - subTable = ((JitRangeSelectPlan*)subPlan)->_index_scan._table; - subIndex = subTable->GetIndex(((JitRangeSelectPlan*)subPlan)->_index_scan._index_id); - } else { - MOT_REPORT_ERROR( - MOT_ERROR_INTERNAL, "JIT Compile", "Invalid sub-plan %u type: %d", i, (int)subPlan->_plan_type); - result = false; - } - if (result && !InitTableInfo(&ctx->m_subQueryTableInfo[i], subTable, subIndex)) { - MOT_REPORT_ERROR(MOT_ERROR_OOM, - "JIT Compile", - "Failed to initialize sub-query table information for code-generation context"); - result = false; - } - if (!result) { - for (uint32_t j = 0; j < i; ++j) { - DestroyTableInfo(&ctx->m_subQueryTableInfo[j]); - } - MOT::MemSessionFree(ctx->m_subQueryTableInfo); - ctx->m_subQueryTableInfo = nullptr; - DestroyTableInfo(&ctx->_table_info); +bool OccTransactionManager::ValidateWriteSet(TxnManager* txMan) +{ + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + for (const auto& raPair : orderedSet) { + const Access* ac = raPair.second; + if (!QuickHeaderValidation(ac)) { return false; } } - return true; } -/** @brief Destroys a compilation context. */ -static void DestroyCodeGenContext(JitTvmCodeGenContext* ctx) +RC OccTransactionManager::LockHeaders(TxnManager* txMan, uint32_t& numSentinelsLock) { - if (ctx != nullptr) { - DestroyTableInfo(&ctx->_table_info); - DestroyTableInfo(&ctx->m_innerTable_info); - for (uint32_t i = 0; i < ctx->m_subQueryCount; ++i) { - DestroyTableInfo(&ctx->m_subQueryTableInfo[i]); + RC rc = RC_OK; + uint64_t sleepTime = 1; + uint64_t thdId = txMan->GetThdId(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + numSentinelsLock = 0; + if (m_validationNoWait) { + while (numSentinelsLock != m_writeSetSize) { + for (const auto& raPair : orderedSet) { + const Access* ac = raPair.second; + Sentinel* sent = ac->m_origSentinel; + if (!sent->TryLock(thdId)) { + break; + } + numSentinelsLock++; + // New insert row is already committed! + // Check if row has changed in sentinel + if (!QuickHeaderValidation(ac)) { + rc = RC_ABORT; + goto final; + } + } + + if (numSentinelsLock != m_writeSetSize) { + ReleaseHeaderLocks(txMan, numSentinelsLock); + numSentinelsLock = 0; + if (m_preAbort) { + for (const auto& acPair : orderedSet) { + const Access* ac = acPair.second; + if (!QuickHeaderValidation(ac)) { + return RC_ABORT; + } + } + } + if (!MOTEngine::GetInstance()->IsRecovering()) { + if (sleepTime > LOCK_TIME_OUT) { + return RC_ABORT; + } else { + if (!IsHighContention()) { + CpuCyclesLevelTime::Sleep(5); + } else { + (void)usleep(m_dynamicSleep); + } + sleepTime = sleepTime << 1; + } + } else { + (void)usleep(1000); + } + } } - if (ctx->m_constValues != nullptr) { - MOT::MemSessionFree(ctx->m_constValues); + } else { + for (const auto& raPair : orderedSet) { + const Access* ac = raPair.second; + Sentinel* sent = ac->m_origSentinel; + sent->Lock(thdId); + numSentinelsLock++; + // New insert row is already committed! + // Check if row has changed in sentinel + if (!QuickHeaderValidation(ac)) { + rc = RC_ABORT; + goto final; + } } } +final: + return rc; } -extern int AllocateConstId(JitTvmCodeGenContext* ctx, int type, Datum value, bool isNull) +bool OccTransactionManager::PreAllocStableRow(TxnManager* txMan) { - int res = -1; - if (ctx->m_constCount == MOT_JIT_MAX_CONST) { - MOT_REPORT_ERROR(MOT_ERROR_RESOURCE_LIMIT, - "JIT Compile", - "Cannot allocate constant identifier, reached limit of %u", - ctx->m_constCount); - } else { - res = ctx->m_constCount++; - ctx->m_constValues[res].consttype = type; - ctx->m_constValues[res].constvalue = value; - ctx->m_constValues[res].constisnull = isNull; + if (GetGlobalConfiguration().m_enableCheckpoint) { + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD || (access->m_type == INS && access->m_params.IsUpgradeInsert() == false)) { + continue; + } + if (access->m_params.IsPrimarySentinel()) { + if (!GetCheckpointManager()->PreAllocStableRow(txMan, access->GetRowFromHeader(), access->m_type)) { + GetCheckpointManager()->FreePreAllocStableRows(txMan); + return false; + } + } + } } + return true; +} + +bool OccTransactionManager::ReserveGcMemory(TxnManager* txMan, const GcMaintenanceInfo& gcMemoryReserve) +{ + bool res = true; + GcManager* gc_manager = txMan->GetGcSession(); + MOT_ASSERT(gc_manager != nullptr); + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::DELETE_QUEUE, gcMemoryReserve.m_delete_queue); + if (!res) { + return false; + } + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::VERSION_QUEUE, gcMemoryReserve.m_version_queue); + if (!res) { + return false; + } + res = + gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::UPDATE_COLUMN_QUEUE, gcMemoryReserve.m_update_column_queue); + if (!res) { + return false; + } + res = gc_manager->ReserveGCMemoryPerQueue(GC_QUEUE_TYPE::GENERIC_QUEUE, gcMemoryReserve.m_generic_queue, true); + if (!res) { + return false; + } + return res; } - -static JitContext* FinalizeCodegen(JitTvmCodeGenContext* ctx, int max_arg, JitCommandType command_type) -{ - // do minimal verification and wrap up - if (!ctx->m_jittedQuery->finalize()) { - MOT_LOG_ERROR("Failed to generate jitted code for query: Failed to finalize jit function"); - ctx->m_jittedQuery->dump(); - delete ctx->m_jittedQuery; - return nullptr; - } - - // dump if requested - if (IsMotCodegenPrintEnabled()) { - ctx->m_jittedQuery->dump(); - } - - // verify function structure - if (!ctx->m_jittedQuery->verify()) { - MOT_LOG_TRACE("Failed to generate jitted code for query: Failed to verify jit function"); - delete ctx->m_jittedQuery; - return nullptr; - } - - // prepare global constant array - JitDatumArray datumArray = {}; - if (ctx->m_constCount > 0) { - if (!PrepareDatumArray(ctx->m_constValues, ctx->m_constCount, &datumArray)) { - MOT_LOG_ERROR("Failed to generate jitted code for query: Failed to prepare constant datum array"); - delete ctx->m_jittedQuery; - return nullptr; - } - } - - // that's it, we are ready - JitContext* jit_context = AllocJitContext(JIT_CONTEXT_GLOBAL); - if (jit_context == nullptr) { - MOT_LOG_TRACE("Failed to allocate JIT context, aborting code generation"); - delete ctx->m_jittedQuery; - return nullptr; - } - - // setup execution details - jit_context->m_table = ctx->_table_info.m_table; - jit_context->m_index = ctx->_table_info.m_index; - jit_context->m_indexId = jit_context->m_index->GetExtId(); - MOT_LOG_TRACE("Installed index id: %" PRIu64, jit_context->m_indexId); - jit_context->m_tvmFunction = ctx->m_jittedQuery; - jit_context->m_argCount = max_arg + 1; - jit_context->m_innerTable = ctx->m_innerTable_info.m_table; - jit_context->m_innerIndex = ctx->m_innerTable_info.m_index; - if (jit_context->m_innerIndex != nullptr) { - jit_context->m_innerIndexId = jit_context->m_innerIndex->GetExtId(); - MOT_LOG_TRACE("Installed inner index id: %" PRIu64, jit_context->m_innerIndexId); - } - jit_context->m_commandType = command_type; - jit_context->m_subQueryCount = 0; - jit_context->m_constDatums.m_datumCount = datumArray.m_datumCount; - jit_context->m_constDatums.m_datums = datumArray.m_datums; - - return jit_context; -} - -static JitContext* JitUpdateCodegen(const Query* query, const char* query_string, JitUpdatePlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT update at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* table = plan->_query._table; - if (!InitCodeGenContext(&cg_ctx, &builder, table, table->GetPrimaryIndex())) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedUpdate", query_string); - IssueDebugLog("Starting execution of jitted UPDATE"); - - // update is not allowed if we reached soft memory limit - buildIsSoftMemoryLimitReached(ctx); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // begin the WHERE clause (this is a point query - int max_arg = 0; - if (!buildPointScan(ctx, &plan->_query._search_exprs, &max_arg, JIT_RANGE_SCAN_MAIN, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for update query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // fetch row for writing - MOT_LOG_DEBUG("Generating update code for point query"); - Instruction* row = buildSearchRow(ctx, MOT::AccessType::WR, JIT_RANGE_SCAN_MAIN); - - // check for additional filters - if (!buildFilterRow(ctx, row, &plan->_query._filters, &max_arg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for update query: unsupported filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // prepare a bitmap array - IssueDebugLog("Resetting bitmap set for incremental redo"); - AddResetBitmapSet(ctx); - - // now begin updating columns - IssueDebugLog("Updating row columns"); - if (!writeRowColumns(ctx, row, &plan->_update_exprs, &max_arg, true)) { - MOT_LOG_TRACE("Failed to generate jitted code for update query: failed to process target entry"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // write row - IssueDebugLog("Writing row"); - buildWriteRow(ctx, row, true, nullptr); - - // the next call will be executed only if the previous call to writeRow succeeded - buildIncrementRowsProcessed(ctx); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_UPDATE); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -static JitContext* JitRangeUpdateCodegen(const Query* query, const char* query_string, JitRangeUpdatePlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT range update at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* table = plan->_index_scan._table; - if (!InitCodeGenContext(&cg_ctx, &builder, table, table->GetPrimaryIndex())) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedRangeUpdate", query_string); - IssueDebugLog("Starting execution of jitted range UPDATE"); - - // update is not allowed if we reached soft memory limit - buildIsSoftMemoryLimitReached(ctx); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // begin the WHERE clause - int max_arg = 0; - MOT_LOG_DEBUG("Generating range cursor for range UPDATE query"); - JitTvmRuntimeCursor cursor = - buildRangeCursor(ctx, &plan->_index_scan, &max_arg, JIT_RANGE_SCAN_MAIN, JIT_INDEX_SCAN_FORWARD, nullptr); - if (cursor.begin_itr == nullptr) { - MOT_LOG_TRACE("Failed to generate jitted code for range UPDATE query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddResetBitmapSet(ctx); - - JIT_WHILE_BEGIN(cursor_loop) - Instruction* res = AddIsScanEnd(ctx, JIT_INDEX_SCAN_FORWARD, &cursor, JIT_RANGE_SCAN_MAIN); - JIT_WHILE_EVAL_NOT(res) - Instruction* row = buildGetRowFromIterator( - ctx, JIT_WHILE_POST_BLOCK(), MOT::AccessType::WR, JIT_INDEX_SCAN_FORWARD, &cursor, JIT_RANGE_SCAN_MAIN); - - // check for additional filters - if (!buildFilterRow(ctx, row, &plan->_index_scan._filters, &max_arg, JIT_WHILE_COND_BLOCK())) { - MOT_LOG_TRACE("Failed to generate jitted code for range UPDATE query: unsupported filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // now begin updating columns - IssueDebugLog("Updating row columns"); - if (!writeRowColumns(ctx, row, &plan->_update_exprs, &max_arg, true)) { - MOT_LOG_TRACE("Failed to generate jitted code for update query: failed to process target entry"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // write row - IssueDebugLog("Writing row"); - buildWriteRow(ctx, row, false, &cursor); - - // the next call will be executed only if the previous call to writeRow succeeded - buildIncrementRowsProcessed(ctx); - - // reset bitmap for next loop - AddResetBitmapSet(ctx); - JIT_WHILE_END() - - // cleanup - IssueDebugLog("Reached end of range update loop"); - AddDestroyCursor(ctx, &cursor); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_RANGE_UPDATE); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -static JitContext* JitInsertCodegen(const Query* query, const char* query_string, JitInsertPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT insert at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* table = plan->_table; - if (!InitCodeGenContext(&cg_ctx, &builder, table, table->GetPrimaryIndex())) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedInsert", query_string); - IssueDebugLog("Starting execution of jitted INSERT"); - - // insert is not allowed if we reached soft memory limit - buildIsSoftMemoryLimitReached(ctx); - - // create new row and bitmap set - Instruction* row = buildCreateNewRow(ctx); - - // set row null bits - IssueDebugLog("Setting row null bits before insert"); - AddSetRowNullBits(ctx, row); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - IssueDebugLog("Setting row columns"); - int max_arg = 0; - if (!writeRowColumns(ctx, row, &plan->_insert_exprs, &max_arg, false)) { - MOT_LOG_TRACE("Failed to generate jitted code for insert query: failed to process target entry"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - IssueDebugLog("Inserting row"); - buildInsertRow(ctx, row); - - // the next call will be executed only if the previous call to writeRow succeeded - buildIncrementRowsProcessed(ctx); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_INSERT); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -static JitContext* JitDeleteCodegen(const Query* query, const char* query_string, JitDeletePlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT delete at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* table = plan->_query._table; - if (!InitCodeGenContext(&cg_ctx, &builder, table, table->GetPrimaryIndex())) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedDelete", query_string); - IssueDebugLog("Starting execution of jitted DELETE"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // begin the WHERE clause - int max_arg = 0; - if (!buildPointScan(ctx, &plan->_query._search_exprs, &max_arg, JIT_RANGE_SCAN_MAIN, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for DELETE query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // fetch row for delete - Instruction* row = buildSearchRow(ctx, MOT::AccessType::DEL, JIT_RANGE_SCAN_MAIN); - - // check for additional filters - if (!buildFilterRow(ctx, row, &plan->_query._filters, &max_arg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for DELETE query: unsupported filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // delete row - IssueDebugLog("Deleting row"); - buildDeleteRow( - ctx); // row is already cached in concurrency control module, so we do not need to provide an argument - - // the next call will be executed only if the previous call to deleteRow succeeded - buildIncrementRowsProcessed(ctx); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_DELETE); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -static JitContext* JitSelectCodegen(const Query* query, const char* query_string, JitSelectPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT select at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* table = plan->_query._table; - if (!InitCodeGenContext(&cg_ctx, &builder, table, table->GetPrimaryIndex())) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedSelect", query_string); - IssueDebugLog("Starting execution of jitted SELECT"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // begin the WHERE clause - int max_arg = 0; - if (!buildPointScan(ctx, &plan->_query._search_exprs, &max_arg, JIT_RANGE_SCAN_MAIN, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for SELECT query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // clear tuple even if row is not found later - AddExecClearTuple(ctx); - - // fetch row for read - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); - - // check for additional filters - if (!buildFilterRow(ctx, row, &plan->_query._filters, &max_arg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for SELECT query: unsupported filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // now begin selecting columns into result - IssueDebugLog("Selecting columns into result"); - if (!selectRowColumns(ctx, row, &plan->_select_exprs, &max_arg, JIT_RANGE_SCAN_MAIN)) { - MOT_LOG_TRACE("Failed to generate jitted code for insert query: failed to process target entry"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddExecStoreVirtualTuple(ctx); - - // update number of rows processed - buildIncrementRowsProcessed(ctx); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended (this is a point query) - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_SELECT); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -static void AddCleanupOldScan(JitTvmCodeGenContext* ctx) -{ - // emit code to cleanup previous scan in case this is a new scan - JIT_IF_BEGIN(cleanup_old_scan) - Instruction* isNewScan = AddIsNewScan(ctx); - JIT_IF_EVAL(isNewScan) - AddDestroyStateIterators(ctx, JIT_RANGE_SCAN_MAIN); - AddDestroyStateIterators(ctx, JIT_RANGE_SCAN_INNER); - // sub-query does not have a stateful execution, so no need to cleanup - JIT_IF_END() -} - -/** @brief Generates code for range SELECT query with a possible LIMIT clause. */ -static JitContext* JitRangeSelectCodegen(const Query* query, const char* query_string, JitRangeSelectPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT select at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* table = plan->_index_scan._table; - int index_id = plan->_index_scan._index_id; - if (!InitCodeGenContext(&cg_ctx, &builder, table, table->GetIndex(index_id))) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedRangeSelect", query_string); - IssueDebugLog("Starting execution of jitted range SELECT"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // clear tuple even if row is not found later - AddExecClearTuple(ctx); - - // emit code to cleanup previous scan in case this is a new scan - AddCleanupOldScan(ctx); - - // prepare stateful scan if not done so already, if no row exists then emit code to return from function - int max_arg = 0; - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* row = buildPrepareStateScanRow( - ctx, &plan->_index_scan, JIT_RANGE_SCAN_MAIN, access_mode, &max_arg, nullptr, nullptr, nullptr); - if (row == nullptr) { - MOT_LOG_TRACE("Failed to generate jitted code for range select query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // select inner and outer row expressions into result tuple (no aggregate because aggregate is not stateful) - IssueDebugLog("Retrieved row from state iterator, beginning to select columns into result tuple"); - if (!selectRowColumns(ctx, row, &plan->_select_exprs, &max_arg, JIT_RANGE_SCAN_MAIN)) { - MOT_LOG_TRACE("Failed to generate jitted code for range SELECT query: failed to select row expressions"); - DestroyCodeGenContext(ctx); - return nullptr; - } - AddExecStoreVirtualTuple(ctx); - buildIncrementRowsProcessed(ctx); - - // make sure that next iteration find an empty state row, so scan makes a progress - AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); - - // if a limit clause exists, then increment limit counter and check if reached limit - buildCheckLimit(ctx, plan->_limit_count); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitCommandType cmdType = - (plan->_index_scan._scan_type == JIT_INDEX_SCAN_FULL) ? JIT_COMMAND_FULL_SELECT : JIT_COMMAND_RANGE_SELECT; - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, cmdType); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -/** @brief Generates code for range SELECT query with aggregator. */ -static JitContext* JitAggregateRangeSelectCodegen( - const Query* query, const char* query_string, JitRangeSelectPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT aggregate range select at thread %p", (void*)pthread_self()); - - Builder builder; - MOT::Table* table = plan->_index_scan._table; - int index_id = plan->_index_scan._index_id; - JitTvmCodeGenContext cg_ctx = {0}; - if (!InitCodeGenContext(&cg_ctx, &builder, table, table->GetIndex(index_id))) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedAggregateRangeSelect", query_string); - IssueDebugLog("Starting execution of jitted aggregate range SELECT"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // clear tuple (we use tuple's resno column as aggregated sum instead of defining local variable) - AddExecClearTuple(ctx); - - // prepare for aggregation - if (!prepareAggregate(ctx, &plan->_aggregate)) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range SELECT query: failed to prepare aggregate"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddResetStateLimitCounter(ctx); - - // pay attention: aggregated range scan is not stateful, since we scan all tuples in one call - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - - // begin the WHERE clause - int max_arg = 0; - JitIndexScanDirection index_scan_direction = JIT_INDEX_SCAN_FORWARD; - - // build range iterators - MOT_LOG_DEBUG("Generating range cursor for range SELECT query"); - JitTvmRuntimeCursor cursor = - buildRangeCursor(ctx, &plan->_index_scan, &max_arg, JIT_RANGE_SCAN_MAIN, index_scan_direction, nullptr); - if (cursor.begin_itr == nullptr) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range SELECT query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - JIT_WHILE_BEGIN(cursor_aggregate_loop) - Instruction* res = AddIsScanEnd(ctx, index_scan_direction, &cursor, JIT_RANGE_SCAN_MAIN); - JIT_WHILE_EVAL_NOT(res) - Instruction* row = buildGetRowFromIterator( - ctx, JIT_WHILE_POST_BLOCK(), access_mode, JIT_INDEX_SCAN_FORWARD, &cursor, JIT_RANGE_SCAN_MAIN); - - // check for additional filters, if not try to fetch next row - if (!buildFilterRow(ctx, row, &plan->_index_scan._filters, &max_arg, JIT_WHILE_COND_BLOCK())) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range SELECT query: unsupported filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // aggregate into tuple (we use tuple's resno column as aggregated sum instead of defining local variable) - // if row disqualified due to DISTINCT operator then go back to loop test block - if (!buildAggregateRow(ctx, &plan->_aggregate, row, JIT_WHILE_COND_BLOCK())) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range SELECT query: unsupported aggregate"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // if a limit clause exists, then increment limit counter and check if reached limit - if (plan->_limit_count > 0) { - AddIncrementStateLimitCounter(ctx); - JIT_IF_BEGIN(limit_count_reached) - Instruction* current_limit_count = AddGetStateLimitCounter(ctx); - JIT_IF_EVAL_CMP(current_limit_count, JIT_CONST(plan->_limit_count), JIT_ICMP_EQ); - IssueDebugLog("Reached limit specified in limit clause, raising internal state scan end flag"); - JIT_WHILE_BREAK() // break from loop - JIT_IF_END() - } - JIT_WHILE_END() - - // cleanup - IssueDebugLog("Reached end of aggregate range select loop"); - AddDestroyCursor(ctx, &cursor); - - // wrap up aggregation and write to result tuple - buildAggregateResult(ctx, &plan->_aggregate); - - // store the result tuple - AddExecStoreVirtualTuple(ctx); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended (this is an aggregate loop) - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_AGGREGATE_RANGE_SELECT); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -static JitContext* JitPointJoinCodegen(const Query* query, const char* query_string, JitJoinPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT Point JOIN query at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* outer_table = plan->_outer_scan._table; - MOT::Index* outer_index = outer_table->GetIndex(plan->_outer_scan._index_id); - MOT::Table* inner_table = plan->_inner_scan._table; - MOT::Index* inner_index = inner_table->GetIndex(plan->_inner_scan._index_id); - if (!InitCodeGenContext(&cg_ctx, &builder, outer_table, outer_index, inner_table, inner_index)) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedPointJoin", query_string); - IssueDebugLog("Starting execution of jitted Point JOIN"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // search the outer row - int max_arg = 0; - if (!buildPointScan(ctx, &plan->_outer_scan._search_exprs, &max_arg, JIT_RANGE_SCAN_MAIN, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for Point JOIN query: unsupported outer WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // clear tuple even if row is not found later - AddExecClearTuple(ctx); - - // fetch row for read - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* outer_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); - - // check for additional filters - if (!buildFilterRow(ctx, outer_row, &plan->_outer_scan._filters, &max_arg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for Point JOIN query: unsupported outer scan filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // before we move on to inner point scan, we save the outer row in a safe copy (but for that purpose we need to save - // row in outer scan state) - AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); - AddCopyOuterStateRow(ctx); - - // now search the inner row - if (!buildPointScan(ctx, &plan->_inner_scan._search_exprs, &max_arg, JIT_RANGE_SCAN_INNER, outer_row)) { - MOT_LOG_TRACE("Failed to generate jitted code for Point JOIN query: unsupported inner WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - Instruction* inner_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_INNER); - - // check for additional filters - if (!buildFilterRow(ctx, inner_row, &plan->_inner_scan._filters, &max_arg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for Point JOIN query: unsupported inner scan filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // retrieve the safe copy of the outer row - Instruction* outer_row_copy = AddGetOuterStateRowCopy(ctx); - - // now begin selecting columns into result - if (!selectJoinRows(ctx, outer_row_copy, inner_row, plan, &max_arg)) { - MOT_LOG_TRACE( - "Failed to generate jitted code for Point JOIN query: failed to select row columns into result tuple"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddExecStoreVirtualTuple(ctx); - - // update number of rows processed - buildIncrementRowsProcessed(ctx); - - // generate code for setting output parameter tp_processed value to rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended (this is a point query) - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_POINT_JOIN); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -/** @brief Generates code for range JOIN query with outer point query and a possible LIMIT clause but without - * aggregation. */ -static JitContext* JitPointOuterJoinCodegen(const Query* query, const char* query_string, JitJoinPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT Outer Point JOIN query at thread %p", (void*)pthread_self()); - - int max_arg = 0; - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* outer_table = plan->_outer_scan._table; - MOT::Index* outer_index = outer_table->GetIndex(plan->_outer_scan._index_id); - MOT::Table* inner_table = plan->_inner_scan._table; - MOT::Index* inner_index = inner_table->GetIndex(plan->_inner_scan._index_id); - if (!InitCodeGenContext(&cg_ctx, &builder, outer_table, outer_index, inner_table, inner_index)) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedOuterPointJoin", query_string); - IssueDebugLog("Starting execution of jitted Outer Point JOIN"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // clear tuple even if row is not found later - AddExecClearTuple(ctx); - - // emit code to cleanup previous scan in case this is a new scan - AddCleanupOldScan(ctx); - - // we first check if outer state row was already searched - Instruction* outer_row_copy = AddGetOuterStateRowCopy(ctx); - JIT_IF_BEGIN(check_outer_row_ready) - JIT_IF_EVAL_NOT(outer_row_copy) - // search the outer row - if (!buildPointScan(ctx, &plan->_outer_scan._search_exprs, &max_arg, JIT_RANGE_SCAN_MAIN, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for Outer Point JOIN query: unsupported outer WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // fetch row for read - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* outer_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); - - // check for additional filters - if (!buildFilterRow(ctx, outer_row, &plan->_outer_scan._filters, &max_arg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for Outer Point JOIN query: unsupported filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // before we move on to inner range scan, we save the outer row in a safe copy (but for that purpose we need to save - // row in outer scan state) - AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); - AddCopyOuterStateRow(ctx); - outer_row_copy = AddGetOuterStateRowCopy(ctx); // must get copy again, otherwise it is null - JIT_IF_END() - - // now prepare inner scan if needed, if no row was found then emit code to return from function (since outer scan is - // a point query) - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* inner_row = buildPrepareStateScanRow( - ctx, &plan->_inner_scan, JIT_RANGE_SCAN_INNER, access_mode, &max_arg, outer_row_copy, nullptr, nullptr); - if (inner_row == nullptr) { - MOT_LOG_TRACE("Failed to generate jitted code for Outer Point JOIN query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // retrieve the safe copy of the outer row - outer_row_copy = AddGetOuterStateRowCopy(ctx); - - // now begin selecting columns into result - if (!selectJoinRows(ctx, outer_row_copy, inner_row, plan, &max_arg)) { - MOT_LOG_TRACE("Failed to generate jitted code for Outer Point JOIN query: failed to select row columns into " - "result tuple"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddExecStoreVirtualTuple(ctx); - - // update number of rows processed - buildIncrementRowsProcessed(ctx); - - // clear inner row for next iteration - AddResetStateRow(ctx, JIT_RANGE_SCAN_INNER); - - // if a limit clause exists, then increment limit counter and check if reached limit - buildCheckLimit(ctx, plan->_limit_count); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_RANGE_JOIN); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -/** @brief Generates code for range JOIN query with inner point query and a possible LIMIT clause but without - * aggregation. */ -static JitContext* JitPointInnerJoinCodegen(const Query* query, const char* query_string, JitJoinPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT Inner Point JOIN at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* outer_table = plan->_outer_scan._table; - MOT::Index* outer_index = outer_table->GetIndex(plan->_outer_scan._index_id); - MOT::Table* inner_table = plan->_inner_scan._table; - MOT::Index* inner_index = inner_table->GetIndex(plan->_inner_scan._index_id); - if (!InitCodeGenContext(&cg_ctx, &builder, outer_table, outer_index, inner_table, inner_index)) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedInnerPointJoin", query_string); - IssueDebugLog("Starting execution of jitted inner point JOIN"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // clear tuple even if row is not found later - AddExecClearTuple(ctx); - - // emit code to cleanup previous scan in case this is a new scan - AddCleanupOldScan(ctx); - - // prepare stateful scan if not done so already, if row not found then emit code to return from function (since this - // is an outer scan) - int max_arg = 0; - BasicBlock* fetch_outer_row_bb = nullptr; - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* outer_row = buildPrepareStateScanRow( - ctx, &plan->_outer_scan, JIT_RANGE_SCAN_MAIN, access_mode, &max_arg, nullptr, nullptr, &fetch_outer_row_bb); - if (outer_row == nullptr) { - MOT_LOG_TRACE("Failed to generate jitted code for Inner Point JOIN query: unsupported WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // before we move on to inner scan, we save the outer row in a safe copy - AddCopyOuterStateRow(ctx); - - // now search the inner row - if (!buildPointScan(ctx, &plan->_inner_scan._search_exprs, &max_arg, JIT_RANGE_SCAN_INNER, outer_row)) { - MOT_LOG_TRACE("Failed to generate jitted code for Inner Point JOIN query: unsupported inner WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - Instruction* inner_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_INNER); - - // check for additional filters - if (!buildFilterRow(ctx, inner_row, &plan->_inner_scan._filters, &max_arg, fetch_outer_row_bb)) { - MOT_LOG_TRACE("Failed to generate jitted code for Inner Point JOIN query: unsupported inner scan filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // retrieve the safe copy of the outer row - Instruction* outer_row_copy = AddGetOuterStateRowCopy(ctx); - - // now begin selecting columns into result - if (!selectJoinRows(ctx, outer_row_copy, inner_row, plan, &max_arg)) { - MOT_LOG_TRACE("Failed to generate jitted code for Inner Point JOIN query: failed to select row columns into " - "result tuple"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddExecStoreVirtualTuple(ctx); - buildIncrementRowsProcessed(ctx); - - // make sure that next iteration find an empty state row, so scan makes a progress - AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); - - // if a limit clause exists, then increment limit counter and check if reached limit - buildCheckLimit(ctx, plan->_limit_count); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_RANGE_JOIN); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -/** @brief Generates code for range JOIN query with a possible LIMIT clause but without aggregation. */ -static JitContext* JitRangeJoinCodegen(const Query* query, const char* query_string, JitJoinPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT range JOIN at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* outer_table = plan->_outer_scan._table; - MOT::Index* outer_index = outer_table->GetIndex(plan->_outer_scan._index_id); - MOT::Table* inner_table = plan->_inner_scan._table; - MOT::Index* inner_index = inner_table->GetIndex(plan->_inner_scan._index_id); - if (!InitCodeGenContext(&cg_ctx, &builder, outer_table, outer_index, inner_table, inner_index)) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedRangeJoin", query_string); - IssueDebugLog("Starting execution of jitted range JOIN"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // clear tuple even if row is not found later - AddExecClearTuple(ctx); - - // emit code to cleanup previous scan in case this is a new scan - AddCleanupOldScan(ctx); - - // prepare stateful scan if not done so already - int max_arg = 0; - BasicBlock* fetch_outer_row_bb = nullptr; - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* outer_row = buildPrepareStateScanRow( - ctx, &plan->_outer_scan, JIT_RANGE_SCAN_MAIN, access_mode, &max_arg, nullptr, nullptr, &fetch_outer_row_bb); - if (outer_row == nullptr) { - MOT_LOG_TRACE("Failed to generate jitted code for Range JOIN query: unsupported outer WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // before we move on to inner scan, we save the outer row in a safe copy - AddCopyOuterStateRow(ctx); - - // now prepare inner scan if needed - Instruction* inner_row = buildPrepareStateScanRow( - ctx, &plan->_inner_scan, JIT_RANGE_SCAN_INNER, access_mode, &max_arg, outer_row, fetch_outer_row_bb, nullptr); - if (inner_row == nullptr) { - MOT_LOG_TRACE("Failed to generate jitted code for Range JOIN query: unsupported inner WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // retrieve the safe copy of the outer row - Instruction* outer_row_copy = AddGetOuterStateRowCopy(ctx); - - // now begin selecting columns into result - if (!selectJoinRows(ctx, outer_row_copy, inner_row, plan, &max_arg)) { - MOT_LOG_TRACE( - "Failed to generate jitted code for Range JOIN query: failed to select row columns into result tuple"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddExecStoreVirtualTuple(ctx); - buildIncrementRowsProcessed(ctx); - - // clear inner row for next iteration - AddResetStateRow(ctx, JIT_RANGE_SCAN_INNER); - - // if a limit clause exists, then increment limit counter and check if reached limit - buildCheckLimit(ctx, plan->_limit_count); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, max_arg, JIT_COMMAND_RANGE_JOIN); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -/** @brief Generates code for range JOIN query with an aggregator. */ -static JitContext* JitAggregateRangeJoinCodegen(const Query* query, const char* query_string, JitJoinPlan* plan) -{ - MOT_LOG_DEBUG("Generating code for MOT aggregate range JOIN at thread %p", (void*)pthread_self()); - - Builder builder; - - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* outer_table = plan->_outer_scan._table; - MOT::Index* outer_index = outer_table->GetIndex(plan->_outer_scan._index_id); - MOT::Table* inner_table = plan->_inner_scan._table; - MOT::Index* inner_index = inner_table->GetIndex(plan->_inner_scan._index_id); - if (!InitCodeGenContext(&cg_ctx, &builder, outer_table, outer_index, inner_table, inner_index)) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedAggregateRangeJoin", query_string); - IssueDebugLog("Starting execution of jitted aggregate range JOIN"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // clear tuple (we use tuple's resno column as aggregated sum instead of defining local variable) - AddExecClearTuple(ctx); - - // prepare for aggregation - if (!prepareAggregate(ctx, &plan->_aggregate)) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range JOIN query: failed to prepare aggregate"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - AddResetStateLimitCounter(ctx); - - // pay attention: aggregated range scan is not stateful, since we scan all tuples in one call - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - - // begin the WHERE clause - int maxArg = 0; - - // build range iterators - MOT_LOG_DEBUG("Generating outer loop cursor for range JOIN query"); - JitTvmRuntimeCursor outer_cursor = - buildRangeCursor(ctx, &plan->_outer_scan, &maxArg, JIT_RANGE_SCAN_MAIN, JIT_INDEX_SCAN_FORWARD, nullptr); - if (outer_cursor.begin_itr == nullptr) { - MOT_LOG_TRACE( - "Failed to generate jitted code for aggregate range JOIN query: unsupported outer-loop WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - JIT_WHILE_BEGIN(cursor_aggregate_outer_loop) - BasicBlock* endOuterLoopBlock = JIT_WHILE_POST_BLOCK(); - Instruction* res = AddIsScanEnd(ctx, JIT_INDEX_SCAN_FORWARD, &outer_cursor, JIT_RANGE_SCAN_MAIN); - JIT_WHILE_EVAL_NOT(res) - Instruction* outer_row = buildGetRowFromIterator( - ctx, JIT_WHILE_POST_BLOCK(), access_mode, JIT_INDEX_SCAN_FORWARD, &outer_cursor, JIT_RANGE_SCAN_MAIN); - - // check for additional filters, if not try to fetch next row - if (!buildFilterRow(ctx, outer_row, &plan->_outer_scan._filters, &maxArg, JIT_WHILE_COND_BLOCK())) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range JOIN query: unsupported outer-loop filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // before we move on to inner scan, we save the outer row in a safe copy (but for that purpose we need to save row - // in outer scan state) - AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); - AddCopyOuterStateRow(ctx); - - // now build the inner loop - MOT_LOG_DEBUG("Generating inner loop cursor for range JOIN query"); - JitTvmRuntimeCursor inner_cursor = - buildRangeCursor(ctx, &plan->_inner_scan, &maxArg, JIT_RANGE_SCAN_INNER, JIT_INDEX_SCAN_FORWARD, outer_row); - if (inner_cursor.begin_itr == nullptr) { - MOT_LOG_TRACE( - "Failed to generate jitted code for aggregate range JOIN query: unsupported inner-loop WHERE clause type"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - JIT_WHILE_BEGIN(cursor_aggregate_inner_loop) - Instruction* res_inner = AddIsScanEnd(ctx, JIT_INDEX_SCAN_FORWARD, &inner_cursor, JIT_RANGE_SCAN_INNER); - JIT_WHILE_EVAL_NOT(res_inner) - Instruction* inner_row = buildGetRowFromIterator( - ctx, JIT_WHILE_POST_BLOCK(), access_mode, JIT_INDEX_SCAN_FORWARD, &inner_cursor, JIT_RANGE_SCAN_INNER); - - // check for additional filters, if not try to fetch next row - if (!buildFilterRow(ctx, inner_row, &plan->_inner_scan._filters, &maxArg, JIT_WHILE_COND_BLOCK())) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range JOIN query: unsupported inner-loop filter"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // aggregate into tuple (we use tuple's resno column as aggregated sum instead of defining local variable) - // find out to which table the aggreate expression refers, and aggregate it - // if row disqualified due to DISTINCT operator then go back to inner loop test block - bool aggRes = false; - if (plan->_aggregate._table == ctx->m_innerTable_info.m_table) { - aggRes = buildAggregateRow(ctx, &plan->_aggregate, inner_row, JIT_WHILE_COND_BLOCK()); - } else { - // retrieve the safe copy of the outer row - Instruction* outer_row_copy = AddGetOuterStateRowCopy(ctx); - aggRes = buildAggregateRow(ctx, &plan->_aggregate, outer_row_copy, JIT_WHILE_COND_BLOCK()); - } - - if (!aggRes) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range JOIN query: unsupported aggregate"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // if a limit clause exists, then increment limit counter and check if reached limit - if (plan->_limit_count > 0) { - AddIncrementStateLimitCounter(ctx); - JIT_IF_BEGIN(limit_count_reached) - Instruction* currentLimitCount = AddGetStateLimitCounter(ctx); - JIT_IF_EVAL_CMP(currentLimitCount, JIT_CONST(plan->_limit_count), JIT_ICMP_EQ); - IssueDebugLog("Reached limit specified in limit clause, raising internal state scan end flag"); - AddDestroyCursor(ctx, &outer_cursor); - AddDestroyCursor(ctx, &inner_cursor); - ctx->_builder->CreateBr(endOuterLoopBlock); // break from inner outside of outer loop - JIT_IF_END() - } - JIT_WHILE_END() - - // cleanup - IssueDebugLog("Reached end of inner loop"); - AddDestroyCursor(ctx, &inner_cursor); - - JIT_WHILE_END() - - // cleanup - IssueDebugLog("Reached end of outer loop"); - AddDestroyCursor(ctx, &outer_cursor); - - // wrap up aggregation and write to result tuple - buildAggregateResult(ctx, &plan->_aggregate); - - // store the result tuple - AddExecStoreVirtualTuple(ctx); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended - AddSetScanEnded(ctx, 1); - - // return success from calling function - builder.CreateRet(builder.CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - JitContext* jit_context = FinalizeCodegen(ctx, maxArg, JIT_COMMAND_AGGREGATE_JOIN); - - // cleanup - DestroyCodeGenContext(ctx); - - return jit_context; -} - -static JitContext* JitJoinCodegen(Query* query, const char* query_string, JitJoinPlan* plan) -{ - JitContext* jit_context = nullptr; - - if (plan->_aggregate._aggreaget_op == JIT_AGGREGATE_NONE) { - switch (plan->_scan_type) { - case JIT_JOIN_SCAN_POINT: - // special case: this is really a point query - jit_context = JitPointJoinCodegen(query, query_string, plan); - break; - - case JIT_JOIN_SCAN_OUTER_POINT: - // special case: outer scan is really a point query - jit_context = JitPointOuterJoinCodegen(query, query_string, plan); - break; - - case JIT_JOIN_SCAN_INNER_POINT: - // special case: inner scan is really a point query - jit_context = JitPointInnerJoinCodegen(query, query_string, plan); - break; - - case JIT_JOIN_SCAN_RANGE: - jit_context = JitRangeJoinCodegen(query, query_string, plan); - break; - - default: - MOT_LOG_TRACE( - "Cannot generate jitteed code for JOIN plan: Invalid JOIN scan type %d", (int)plan->_scan_type); - break; - } - } else { - jit_context = JitAggregateRangeJoinCodegen(query, query_string, plan); - } - - return jit_context; -} - -static bool JitSubSelectCodegen(JitTvmCodeGenContext* ctx, JitCompoundPlan* plan, int subQueryIndex) -{ - MOT_LOG_DEBUG("Generating code for MOT sub-select at thread %p", (intptr_t)pthread_self()); - IssueDebugLog("Executing simple SELECT sub-query"); - - // get the sub-query plan - JitSelectPlan* subPlan = (JitSelectPlan*)plan->_sub_query_plans[subQueryIndex]; - - // begin the WHERE clause - int maxArg = 0; - if (!buildPointScan( - ctx, &subPlan->_query._search_exprs, &maxArg, JIT_RANGE_SCAN_SUB_QUERY, nullptr, -1, subQueryIndex)) { - MOT_LOG_TRACE("Failed to generate jitted code for SELECT sub-query: unsupported WHERE clause type"); - return false; - } - - // fetch row for read - Instruction* row = buildSearchRow(ctx, MOT::AccessType::RD, JIT_RANGE_SCAN_SUB_QUERY, subQueryIndex); - - // check for additional filters - if (!buildFilterRow(ctx, row, &subPlan->_query._filters, &maxArg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for SELECT sub-query: unsupported filter"); - return false; - } - - // now begin selecting columns into result - IssueDebugLog("Selecting column into result"); - if (!selectRowColumns(ctx, row, &subPlan->_select_exprs, &maxArg, JIT_RANGE_SCAN_SUB_QUERY, subQueryIndex)) { - MOT_LOG_TRACE("Failed to generate jitted code for SELECT sub-query: failed to process target entry"); - return false; - } - - return true; -} - -/** @brief Generates code for range SELECT sub-query with aggregator. */ -static bool JitSubAggregateRangeSelectCodegen(JitTvmCodeGenContext* ctx, JitCompoundPlan* plan, int subQueryIndex) -{ - MOT_LOG_DEBUG("Generating code for MOT aggregate range select sub-query at thread %p", (intptr_t)pthread_self()); - IssueDebugLog("Executing aggregated range select sub-query"); - - // get the sub-query plan - JitRangeSelectPlan* subPlan = (JitRangeSelectPlan*)plan->_sub_query_plans[subQueryIndex]; - - // prepare for aggregation - if (!prepareAggregate(ctx, &subPlan->_aggregate)) { - MOT_LOG_TRACE( - "Failed to generate jitted code for aggregate range SELECT sub-query: failed to prepare aggregate"); - return false; - } - - AddResetStateLimitCounter(ctx); - - // pay attention: aggregated range scan is not stateful, since we scan all tuples in one call - MOT::AccessType accessMode = MOT::AccessType::RD; - - // begin the WHERE clause - int maxArg = 0; - JitIndexScanDirection index_scan_direction = JIT_INDEX_SCAN_FORWARD; - - // build range iterators - MOT_LOG_DEBUG("Generating range cursor for range SELECT sub-query"); - JitTvmRuntimeCursor cursor = buildRangeCursor( - ctx, &subPlan->_index_scan, &maxArg, JIT_RANGE_SCAN_SUB_QUERY, index_scan_direction, nullptr, subQueryIndex); - if (cursor.begin_itr == nullptr) { - MOT_LOG_TRACE( - "Failed to generate jitted code for aggregate range SELECT sub-query: unsupported WHERE clause type"); - return false; - } - - JIT_WHILE_BEGIN(cursor_aggregate_loop) - Instruction* res = AddIsScanEnd(ctx, index_scan_direction, &cursor, JIT_RANGE_SCAN_SUB_QUERY, subQueryIndex); - JIT_WHILE_EVAL_NOT(res) - Instruction* row = buildGetRowFromIterator(ctx, - JIT_WHILE_POST_BLOCK(), - accessMode, - JIT_INDEX_SCAN_FORWARD, - &cursor, - JIT_RANGE_SCAN_SUB_QUERY, - subQueryIndex); - - // check for additional filters, if not try to fetch next row - if (!buildFilterRow(ctx, row, &subPlan->_index_scan._filters, &maxArg, JIT_WHILE_COND_BLOCK())) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range SELECT sub-query: unsupported filter"); - return false; - } - - // aggregate into tuple (we use tuple's resno column as aggregated sum instead of defining local variable) - // if row disqualified due to DISTINCT operator then go back to loop test block - if (!buildAggregateRow(ctx, &subPlan->_aggregate, row, JIT_WHILE_COND_BLOCK())) { - MOT_LOG_TRACE("Failed to generate jitted code for aggregate range SELECT sub-query: unsupported aggregate"); - return false; - } - - // if a limit clause exists, then increment limit counter and check if reached limit - if (subPlan->_limit_count > 0) { - AddIncrementStateLimitCounter(ctx); - JIT_IF_BEGIN(limit_count_reached) - Instruction* current_limit_count = AddGetStateLimitCounter(ctx); - JIT_IF_EVAL_CMP(current_limit_count, JIT_CONST(subPlan->_limit_count), JIT_ICMP_EQ); - IssueDebugLog("Reached limit specified in limit clause, raising internal state scan end flag"); - JIT_WHILE_BREAK() // break from loop - JIT_IF_END() - } - JIT_WHILE_END() - - // cleanup - IssueDebugLog("Reached end of aggregate range select sub-query loop"); - AddDestroyCursor(ctx, &cursor); - - // wrap up aggregation and write to result tuple (even though this is unfitting to outer query tuple...) - buildAggregateResult(ctx, &subPlan->_aggregate); - - // coy aggregate result from outer query result tuple into sub-query result tuple - AddCopyAggregateToSubQueryResult(ctx, subQueryIndex); - - return true; -} - -static bool JitSubQueryCodeGen(JitTvmCodeGenContext* ctx, JitCompoundPlan* plan, int subQueryIndex) -{ - bool result = false; - JitPlan* subPlan = plan->_sub_query_plans[subQueryIndex]; - if (subPlan->_plan_type == JIT_PLAN_POINT_QUERY) { - result = JitSubSelectCodegen(ctx, plan, subQueryIndex); - } else if (subPlan->_plan_type == JIT_PLAN_RANGE_SCAN) { - result = JitSubAggregateRangeSelectCodegen(ctx, plan, subQueryIndex); - } else { - MOT_REPORT_ERROR(MOT_ERROR_INVALID_ARG, - "Generate JIT Code", - "Cannot generate JIT code for sub-query plan: Invalid plan type %d", - (int)subPlan->_plan_type); - } - return result; -} - -static JitContext* JitCompoundOuterSelectCodegen( - JitTvmCodeGenContext* ctx, Query* query, const char* query_string, JitSelectPlan* plan) -{ - // begin the WHERE clause - int max_arg = 0; - if (!buildPointScan(ctx, &plan->_query._search_exprs, &max_arg, JIT_RANGE_SCAN_MAIN, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for COMPOUND SELECT query: unsupported WHERE clause type"); - return nullptr; - } - - // fetch row for read - MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; - Instruction* row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); - - // check for additional filters - if (!buildFilterRow(ctx, row, &plan->_query._filters, &max_arg, nullptr)) { - MOT_LOG_TRACE("Failed to generate jitted code for COMPOUND SELECT query: unsupported filter"); - return nullptr; - } - - // now begin selecting columns into result - IssueDebugLog("Selecting columns into result"); - if (!selectRowColumns(ctx, row, &plan->_select_exprs, &max_arg, JIT_RANGE_SCAN_MAIN)) { - MOT_LOG_TRACE("Failed to generate jitted code for COMPOUND SELECT query: failed to process target entry"); - return nullptr; - } - - AddExecStoreVirtualTuple(ctx); - - // update number of rows processed - buildIncrementRowsProcessed(ctx); - - // execute *tp_processed = rows_processed - AddSetTpProcessed(ctx); - - // signal to envelope executor scan ended (this is a point query) - AddSetScanEnded(ctx, 1); - - // return success from calling function - ctx->_builder->CreateRet(ctx->_builder->CreateConst((uint64_t)MOT::RC_OK)); - - // wrap up - return FinalizeCodegen(ctx, max_arg, JIT_COMMAND_COMPOUND_SELECT); -} - -static JitContext* JitCompoundOuterCodegen( - JitTvmCodeGenContext* ctx, Query* query, const char* queryString, JitCompoundPlan* plan) -{ - JitContext* jitContext = nullptr; - if (plan->_command_type == JIT_COMMAND_SELECT) { - jitContext = JitCompoundOuterSelectCodegen(ctx, query, queryString, (JitSelectPlan*)plan->_outer_query_plan); - } - // currently other outer query types are not supported - return jitContext; -} - -static JitContext* JitCompoundCodegen(Query* query, const char* query_string, JitCompoundPlan* plan) -{ - // a compound query plan contains one or more sub-queries that evaluate to a datum that next needs to be fed as a - // parameter to the outer query. We are currently imposing the following limitations: - // 1. one sub-query that can only be a MAX aggregate - // 2. outer query must be a simple point select query. - // - // our main strategy is as follows (based on the fact that each sub-query evaluates into a single value) - // 1. for each sub-query: - // 1.1 execute sub-query and put datum result in sub-query result slot, according to sub-query index - // 2. execute the outer query as a simple query - // 3. whenever we encounter a sub-link expression, it is evaluated as an expression that reads the pre-computed - // sub-query result in step 1.1, according to sub-query index - MOT_LOG_DEBUG("Generating code for MOT compound select at thread %p", (intptr_t)pthread_self()); - - // prepare code generation context - Builder builder; - JitTvmCodeGenContext cg_ctx = {0}; - MOT::Table* table = plan->_outer_query_plan->_query._table; - if (!InitCompoundCodeGenContext(&cg_ctx, &builder, table, table->GetPrimaryIndex(), plan)) { - return nullptr; - } - JitTvmCodeGenContext* ctx = &cg_ctx; - - // prepare the jitted function (declare, get arguments into context and define locals) - CreateJittedFunction(ctx, "MotJittedCompoundSelect", query_string); - IssueDebugLog("Starting execution of jitted COMPOUND SELECT"); - - // initialize rows_processed local variable - buildResetRowsProcessed(ctx); - - // generate code for sub-query execution - uint32_t subQueryCount = 0; - for (int i = 0; i < plan->_outer_query_plan->_query._search_exprs._count; ++i) { - if (plan->_outer_query_plan->_query._search_exprs._exprs[i]._expr->_expr_type == JIT_EXPR_TYPE_SUBLINK) { - JitSubLinkExpr* subLinkExpr = - (JitSubLinkExpr*)plan->_outer_query_plan->_query._search_exprs._exprs[i]._expr; - if (!JitSubQueryCodeGen(ctx, plan, subLinkExpr->_sub_query_index)) { - MOT_LOG_TRACE( - "Failed to generate jitted code for COMPOUND SELECT query: Failed to generate code for sub-query"); - DestroyCodeGenContext(ctx); - return nullptr; - } - ++subQueryCount; - } - } - - // clear tuple early, so that we will have a null datum in case outer query finds nothing - AddExecClearTuple(ctx); - - // generate code for the outer query - JitContext* jitContext = JitCompoundOuterCodegen(ctx, query, query_string, plan); - if (jitContext == nullptr) { - MOT_LOG_TRACE("Failed to generate code for outer query in compound select"); - DestroyCodeGenContext(ctx); - return nullptr; - } - - // prepare sub-query data in resulting JIT context (for later execution) - MOT_ASSERT(subQueryCount > 0); - MOT_ASSERT(subQueryCount == plan->_sub_query_count); - if ((subQueryCount > 0) && !PrepareSubQueryData(jitContext, plan)) { - MOT_LOG_TRACE("Failed to prepare tuple table slot array for sub-queries in JIT context object"); - DestroyJitContext(jitContext); - jitContext = nullptr; - } - - // cleanup - DestroyCodeGenContext(ctx); - - return jitContext; -} - -static JitContext* JitRangeScanCodegen(const Query* query, const char* query_string, JitRangeScanPlan* plan) -{ - JitContext* jit_context = nullptr; - - switch (plan->_command_type) { - case JIT_COMMAND_UPDATE: - jit_context = JitRangeUpdateCodegen(query, query_string, (JitRangeUpdatePlan*)plan); +/* function name:ValidateOcc + function purpose:Verify that all operations done by the transaction are legal before the transaction is committed, + that is, they do not conflict with other concurrent transactions + input:transaction manager + output:Validation results(enum variable). + note:none + annotator:liushifa + annotate time:2023/08/25 10:59:07 + contact:3325287047@qq.com +*/ +RC OccTransactionManager::ValidateOcc(TxnManager* txMan) +{ // Initialize the number of locked sentinels to 0 + uint32_t numSentinelLock = 0; + m_rowsLocked = false; + // Get the access manager for the transaction + TxnAccess* txnAccess = txMan->m_accessMgr; + RC rc = RC_OK; + const uint32_t rowCount = txnAccess->Size(); + + m_writeSetSize = 0; + m_insertSetSize = 0; + m_txnCounter++; +// If no rows are accessed, the transaction is read-only and returns success directly + if (rowCount == 0) { + // READONLY + return rc; + } +// Initialize garbage collection maintenance information + GcMaintenanceInfo gcMemoryReserve{}; + + MOT_ASSERT(rowCount == txnAccess->GetOrderedRowSet().size()); + + do { + /* 1.Perform pre-abort check and pre-processing */ + if (!PreAbortCheck(txMan, gcMemoryReserve)) { + rc = RC_ABORT; break; + } + // try to lock all headers + rc = LockHeaders(txMan, numSentinelLock); + if (rc != RC_OK) { + break; + } + // verify write set + if (!ValidateWriteSet(txMan)) { + rc = RC_ABORT; + break; + } - case JIT_COMMAND_SELECT: { - JitRangeSelectPlan* range_select_plan = (JitRangeSelectPlan*)plan; - if (range_select_plan->_aggregate._aggreaget_op == JIT_AGGREGATE_NONE) { - jit_context = JitRangeSelectCodegen(query, query_string, range_select_plan); + // Pre-allocate stable row according to the checkpoint state. + if (!PreAllocStableRow(txMan)) { + rc = RC_MEMORY_ALLOCATION_ERROR; + break; + } + // Reserve memory for possible garbage collection operations + if (!ReserveGcMemory(txMan, gcMemoryReserve)) { + rc = RC_MEMORY_ALLOCATION_ERROR; + break; + } + } while (0); + // If all the above steps are successful + if (likely(rc == RC_OK)) { + MOT_ASSERT(numSentinelLock == m_writeSetSize); + // mark the row as locked + m_rowsLocked = true; + } else { + // Release all header locks + ReleaseHeaderLocks(txMan, numSentinelLock); + if (likely(rc == RC_ABORT)) { + m_abortsCounter++; + } + } + + return rc; +} + +RC OccTransactionManager::ResolveRecoveryOccConflict(TxnManager* txMan, Access* access) +{ + Row* row = nullptr; + RC rc = RC_ABORT; + uint64_t endCSN = static_cast(-1); + MOT_ASSERT(access->m_type == INS); + switch (access->m_origSentinel->GetIndexOrder()) { + case IndexOrder::INDEX_ORDER_PRIMARY: + // Check what is the current row the sentinel is pointing + row = access->m_origSentinel->GetData(); + if (row) { + // Row must be deleted with Smaller CSN + if (row->IsRowDeleted() == false) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + MOT_ASSERT(access->m_origSentinel->GetData()->GetCommitSequenceNumber() > access->m_csn); + // Reset the global version and set to insert on delete + access->m_params.SetUpgradeInsert(); + access->m_params.SetInsertOnDeletedRow(); + access->m_globalRow = row; + access->m_csn = row->GetCommitSequenceNumber(); + access->m_snapshot = static_cast(-1); + rc = RC_OK; } else { - jit_context = JitAggregateRangeSelectCodegen(query, query_string, range_select_plan); + MOT_ASSERT(false); + return RC_ABORT; } - } break; - - default: - MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, - "Generate JIT Code", - "Invalid point query JIT plan command type %d", - (int)plan->_command_type); + break; + case IndexOrder::INDEX_ORDER_SECONDARY: + endCSN = static_cast(access->m_origSentinel)->GetEndCSN(); + if (txMan->GetCommitSequenceNumber() <= endCSN) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + if (endCSN == Sentinel::SENTINEL_INIT_CSN) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + access->m_params.SetUpgradeInsert(); + access->m_params.SetInsertOnDeletedRow(); + access->m_csn = static_cast(access->m_origSentinel)->GetStartCSN(); + rc = RC_OK; + break; + case IndexOrder::INDEX_ORDER_SECONDARY_UNIQUE: + PrimarySentinelNode* node = static_cast(access->m_origSentinel)->GetTopNode(); + if (node != nullptr) { + if (txMan->GetCommitSequenceNumber() <= node->GetEndCSN()) { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + } + // Reset Visible node + access->m_params.SetUpgradeInsert(); + if (node->GetEndCSN() < Sentinel::SENTINEL_INIT_CSN) { + access->m_params.SetInsertOnDeletedRow(); + } else { + MOT_LOG_ERROR("ERROR In Recovery Order!"); + return RC_ABORT; + } + access->m_secondaryUniqueNode = node; + rc = RC_OK; break; } - - return jit_context; + return rc; } - -static JitContext* JitPointQueryCodegen(const Query* query, const char* query_string, JitPointQueryPlan* plan) +void OccTransactionManager::WriteChanges(TxnManager* txMan) { - JitContext* jit_context = nullptr; - - switch (plan->_command_type) { - case JIT_COMMAND_UPDATE: - jit_context = JitUpdateCodegen(query, query_string, (JitUpdatePlan*)plan); - break; - - case JIT_COMMAND_DELETE: - jit_context = JitDeleteCodegen(query, query_string, (JitDeletePlan*)plan); - break; - - case JIT_COMMAND_SELECT: - jit_context = JitSelectCodegen(query, query_string, (JitSelectPlan*)plan); - break; - - default: - MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, - "Generate JIT Code", - "Invalid point query JIT plan command type %d", - (int)plan->_command_type); - break; + if (m_writeSetSize == 0 && m_insertSetSize == 0) { + return; } - return jit_context; -} + MOTConfiguration& cfg = GetGlobalConfiguration(); + uint64_t commit_csn = txMan->GetCommitSequenceNumber(); + uint64_t transaction_id = txMan->GetInternalTransactionId(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); -extern JitContext* JitCodegenTvmQuery(Query* query, const char* query_string, JitPlan* plan) -{ - JitContext* jit_context = nullptr; - - MOT_LOG_DEBUG("*** Attempting to generate planned TVM-jitted code for query: %s", query_string); - - switch (plan->_plan_type) { - case JIT_PLAN_INSERT_QUERY: - jit_context = JitInsertCodegen(query, query_string, (JitInsertPlan*)plan); - break; - - case JIT_PLAN_POINT_QUERY: - jit_context = JitPointQueryCodegen(query, query_string, (JitPointQueryPlan*)plan); - break; - - case JIT_PLAN_RANGE_SCAN: - jit_context = JitRangeScanCodegen(query, query_string, (JitRangeScanPlan*)plan); - break; - - case JIT_PLAN_JOIN: - jit_context = JitJoinCodegen(query, query_string, (JitJoinPlan*)plan); - break; - - case JIT_PLAN_COMPOUND: - jit_context = JitCompoundCodegen(query, query_string, (JitCompoundPlan*)plan); - break; - - default: - MOT_REPORT_ERROR( - MOT_ERROR_INTERNAL, "Generate JIT Code", "Invalid JIT plan type %d", (int)plan->_plan_type); - break; - } - - if (jit_context == nullptr) { - MOT_LOG_TRACE("Failed to generate TVM-jitted code for query: %s", query_string); - } else { - MOT_LOG_DEBUG( - "Got TVM-jitted function %p after compile, for query: %s", jit_context->m_tvmFunction, query_string); - } - - return jit_context; -} - -extern int JitExecTvmQuery(JitContext* jit_context, ParamListInfo params, TupleTableSlot* slot, uint64_t* tp_processed, - int* scan_ended, int newScan) -{ - int result = 0; - ExecContext* exec_context = jit_context->m_execContext; - - // allocate execution context on-demand - if (exec_context == nullptr) { - exec_context = allocExecContext(jit_context->m_tvmFunction->getRegisterCount()); - if (exec_context == nullptr) { - MOT_REPORT_ERROR(MOT_ERROR_OOM, "Execute JIT", "Failed to allocate execution context for TVM-jit function"); - result = MOT::RC_MEMORY_ALLOCATION_ERROR; - } else { - // save for later execution - jit_context->m_execContext = exec_context; + // Stable rows for checkpoint needs to be created (copied from original row) before modifying the global rows. + if (cfg.m_enableCheckpoint) { + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD) { + continue; + } + if (access->m_params.IsPrimarySentinel()) { + // Pass the actual global row (access->GetRowFromHeader()), so that the stable row will have the + // same CSN, rowid, etc as the original row before the modifications are applied. + GetCheckpointManager()->ApplyWrite(txMan, access->GetRowFromHeader(), access); + } } } - if (exec_context != nullptr) { - exec_context->_jit_context = jit_context; - exec_context->_params = params; - exec_context->_slot = slot; - exec_context->_tp_processed = tp_processed; - exec_context->_scan_ended = scan_ended; - exec_context->m_newScan = newScan; - - result = (int)jit_context->m_tvmFunction->exec(exec_context); + // Update CSN with all relevant information on global rows + // For deletes invalidate sentinels - rows still locked! + for (const auto& raPair : orderedSet) { + Access* access = raPair.second; + access->WriteGlobalChanges(commit_csn, transaction_id); } - return result; + WriteSentinelChanges(txMan); + + // For Recovery operation:Update transactionID + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD) { + continue; + } + if (access->m_params.IsPrimarySentinel()) { + static_cast(access->m_origSentinel)->SetTransactionId(transaction_id); + } + } + + m_isTransactionCommited = true; } -} // namespace JitExec + +void OccTransactionManager::WriteSentinelChanges(TxnManager* txMan) +{ + uint64_t commit_csn = txMan->GetCommitSequenceNumber(); + S_SentinelNodePool* sentinelObjectPool = txMan->m_accessMgr->GetSentinelObjectPool(); + TxnOrderedSet_t& orderedSet = txMan->m_accessMgr->GetOrderedRowSet(); + + if (m_insertSetSize > 0) { + for (const auto& raPair : orderedSet) { + Access* access = raPair.second; + if (access->m_type != INS) { + continue; + } + MOT_ASSERT(access->m_origSentinel->IsLocked() == true); + if (access->m_params.IsUpgradeInsert() == false) { + if (access->m_params.IsPrimarySentinel()) { + MOT_ASSERT(access->m_origSentinel->IsDirty() == true); + MOT_ASSERT(access->m_origSentinel->IsLocked() == true); + // Connect row and sentinel, row is set to absent and locked + access->m_origSentinel->SetStartCSN(commit_csn); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + // Current state: row is set to absent,sentinel is locked and not dirty + // Readers will not see the row + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + access->GetTxnRow()->GetTable()->UpdateRowCount(1); + } else { + if (access->m_params.IsSecondaryUniqueSentinel() == false) { + // We only set the in the secondary sentinel! + access->m_origSentinel->SetStartCSN(commit_csn); + access->m_origSentinel->SetEndCSN(Sentinel::SENTINEL_INIT_CSN); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()->GetPrimarySentinel()); + } else { + access->m_origSentinel->SetStartCSN(commit_csn); + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + access->m_origSentinel->SetNextPtr(node); + } + } + // Unset Dirty - still locked! + access->m_origSentinel->UnSetDirty(); + } else { + if (access->m_params.IsInsertOnDeletedRow()) { + if (access->m_params.IsPrimarySentinel()) { + access->GetTxnRow()->GetTable()->UpdateRowCount(1); + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + } else if (access->m_params.IsUniqueIndex() == true) { + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + auto oldNode = static_cast(access->m_origSentinel)->GetTopNode(); + node->SetNextVersion(oldNode); + access->m_origSentinel->SetNextPtr(node); + } else { + MOT_ASSERT(access->m_params.IsIndexUpdate()); + // Revalidate End CSN + static_cast(access->m_origSentinel)->SetEndCSN(Sentinel::SENTINEL_INIT_CSN); + } + } else { + // We delete row internally and insert a new row + if (access->m_params.IsPrimarySentinel()) { + access->m_origSentinel->SetNextPtr(access->GetLocalInsertRow()); + COMPILER_BARRIER; + access->m_origSentinel->SetWriteBit(); + } else { + auto object = sentinelObjectPool->find(access->m_origSentinel->GetIndex()); + PrimarySentinelNode* node = object->second; + (void)sentinelObjectPool->erase(object); + node->Init( + commit_csn, Sentinel::SENTINEL_INIT_CSN, access->GetLocalInsertRow()->GetPrimarySentinel()); + auto oldNode = static_cast(access->m_origSentinel)->GetTopNode(); + oldNode->SetEndCSN(commit_csn); + node->SetNextVersion(oldNode); + access->m_origSentinel->SetNextPtr(node); + } + } + } + } + } +} +/* function name:ReleaseHeaderLocks + function purpose:Releases any row header locks acquired by the transaction during its execution. + input:transaction managers and number of locks + output:none + note:none + annotator:liushifa + annotate time:2023/08/25 11:19:14 + contact:3325287047@qq.com +*/ +void OccTransactionManager::ReleaseHeaderLocks(TxnManager* txMan, uint32_t numOfLocks) +{ + if (numOfLocks == 0) { + return; + } + + TxnAccess* tx = txMan->m_accessMgr; + TxnOrderedSet_t& orderedSet = tx->GetOrderedRowSet(); + // use local counter to optimize + for (const auto& raPair : orderedSet) { + const Access* access = raPair.second; + if (access->m_type == RD) { + continue; + } else { + numOfLocks--; + access->m_origSentinel->Unlock(); + } + if (!numOfLocks) { + break; + } + } +} + +void OccTransactionManager::CleanUp() +{ + m_writeSetSize = 0; + m_insertSetSize = 0; + m_isTransactionCommited = false; +} +} // namespace MOT diff --git a/src/gausskernel/storage/mot/jit_llvm_query_codegen.cpp b/src/gausskernel/storage/mot/jit_llvm_query_codegen.cpp new file mode 100644 index 000000000..73c90f7a6 --- /dev/null +++ b/src/gausskernel/storage/mot/jit_llvm_query_codegen.cpp @@ -0,0 +1,2666 @@ +/* + * Copyright (c) 2020 Huawei Technologies Co.,Ltd. + * + * openGauss is licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * + * http://license.coscl.org.cn/MulanPSL2 + * + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * ------------------------------------------------------------------------- + * + * jit_llvm_query_codegen.cpp + * LLVM JIT-compiled code generation. + * + * IDENTIFICATION + * src/gausskernel/storage/mot/jit_exec/jit_llvm_query_codegen.cpp + * + * ------------------------------------------------------------------------- + */ + +/* + * ATTENTION: Be sure to include jit_llvm_query.h before anything else because of gscodegen.h + * See jit_llvm_query.h for more details. + */ +#include "jit_llvm_query.h" +#include "jit_llvm_query_codegen.h" +#include "jit_llvm.h" +#include "jit_llvm_funcs.h" +#include "jit_llvm_blocks.h" +#include "jit_util.h" +#include "mot_error.h" +#include "utilities.h" +#include "mm_global_api.h" +#include "jit_source_map.h" + +#ifdef ENABLE_LLVM_COMPILE +// for checking if LLVM_ENABLE_DUMP is defined and for using LLVM_VERSION_STRING +#include "llvm/Config/llvm-config.h" +#endif + +namespace JitExec { +DECLARE_LOGGER(JitLlvmQueryCodegen, JitExec) + +using namespace dorado; + +// forward declarations +static void DestroyCodeGenContext(JitLlvmCodeGenContext* ctx); + +/** @brief Define all LLVM prototypes. */ +void InitCodeGenContextFuncs(JitLlvmCodeGenContext* ctx) +{ + llvm::Module* module = ctx->m_codeGen->module(); + + // define all function calls + defineDebugLog(ctx, module); + defineIsSoftMemoryLimitReached(ctx, module); + defineGetPrimaryIndex(ctx, module); + defineGetTableIndex(ctx, module); + defineInitKey(ctx, module); + defineGetColumnAt(ctx, module); + DefineGetExprIsNull(ctx, module); + DefineSetExprIsNull(ctx, module); + DefineGetExprCollation(ctx, module); + DefineSetExprCollation(ctx, module); + defineGetDatumParam(ctx, module); + defineReadDatumColumn(ctx, module); + defineWriteDatumColumn(ctx, module); + defineBuildDatumKey(ctx, module); + + defineSetBit(ctx, module); + defineResetBitmapSet(ctx, module); + defineGetTableFieldCount(ctx, module); + defineWriteRow(ctx, module); + defineSearchRow(ctx, module); + defineCreateNewRow(ctx, module); + defineInsertRow(ctx, module); + defineDeleteRow(ctx, module); + defineSetRowNullBits(ctx, module); + defineSetExprResultNullBit(ctx, module); + defineExecClearTuple(ctx, module); + defineExecStoreVirtualTuple(ctx, module); + defineSelectColumn(ctx, module); + defineSetTpProcessed(ctx, module); + defineSetScanEnded(ctx, module); + + defineCopyKey(ctx, module); + defineFillKeyPattern(ctx, module); + defineAdjustKey(ctx, module); + defineSearchIterator(ctx, module); + defineBeginIterator(ctx, module); + defineCreateEndIterator(ctx, module); + defineIsScanEnd(ctx, module); + DefineCheckRowExistsInIterator(ctx, module); + defineGetRowFromIterator(ctx, module); + defineDestroyIterator(ctx, module); + + defineSetStateIterator(ctx, module); + defineGetStateIterator(ctx, module); + defineIsStateIteratorNull(ctx, module); + defineIsStateScanEnd(ctx, module); + defineGetRowFromStateIteratorFunc(ctx, module); + defineDestroyStateIterators(ctx, module); + defineSetStateScanEndFlag(ctx, module); + defineGetStateScanEndFlag(ctx, module); + + defineResetStateRow(ctx, module); + defineSetStateRow(ctx, module); + defineGetStateRow(ctx, module); + defineCopyOuterStateRow(ctx, module); + defineGetOuterStateRowCopy(ctx, module); + defineIsStateRowNull(ctx, module); + + defineResetStateLimitCounter(ctx, module); + defineIncrementStateLimitCounter(ctx, module); + defineGetStateLimitCounter(ctx, module); + + definePrepareAvgArray(ctx, module); + defineLoadAvgArray(ctx, module); + defineSaveAvgArray(ctx, module); + defineComputeAvgFromArray(ctx, module); + + defineResetAggValue(ctx, module); + defineGetAggValue(ctx, module); + defineSetAggValue(ctx, module); + defineGetAggValueIsNull(ctx, module); + defineSetAggValueIsNull(ctx, module); + + definePrepareDistinctSet(ctx, module); + defineInsertDistinctItem(ctx, module); + defineDestroyDistinctSet(ctx, module); + + defineResetTupleDatum(ctx, module); + defineReadTupleDatum(ctx, module); + defineWriteTupleDatum(ctx, module); + + DefineSelectSubQueryResultFunc(ctx, module); + DefineCopyAggregateToSubQueryResultFunc(ctx, module); + + DefineGetSubQuerySlot(ctx, module); + DefineGetSubQueryTable(ctx, module); + DefineGetSubQueryIndex(ctx, module); + DefineGetSubQuerySearchKey(ctx, module); + DefineGetSubQueryEndIteratorKey(ctx, module); + DefineGetConstAt(ctx, module); + DefineGetInvokeParamListInfo(ctx, module); + DefineSetParamValue(ctx, module); + DefineInvokeStoredProcedure(ctx, module); + DefineConvertViaString(ctx, module); + + DefineEmitProfileData(ctx, module); +} + +/** @brief Define all LLVM used types synonyms. */ +void InitCodeGenContextTypes(JitLlvmCodeGenContext* ctx) +{ + llvm::LLVMContext& context = ctx->m_codeGen->context(); + + // PG types + ctx->ParamListInfoDataType = llvm::StructType::create(context, "ParamListInfoData"); + ctx->TupleTableSlotType = llvm::StructType::create(context, "TupleTableSlot"); + ctx->NumericDataType = llvm::StructType::create(context, "NumericData"); + ctx->VarCharType = llvm::StructType::create(context, "VarChar"); + ctx->BpCharType = llvm::StructType::create(context, "BpChar"); + + // MOT types + ctx->MMEngineType = llvm::StructType::create(context, "MMEngine"); + ctx->TableType = llvm::StructType::create(context, "Table"); + ctx->IndexType = llvm::StructType::create(context, "Index"); + ctx->KeyType = llvm::StructType::create(context, "Key"); + ctx->ColumnType = llvm::StructType::create(context, "Column"); + ctx->RowType = llvm::StructType::create(context, "Row"); + ctx->BitmapSetType = llvm::StructType::create(context, "BitmapSet"); + ctx->IndexIteratorType = llvm::StructType::create(context, "IndexIterator"); +} + +/** @brief Initializes a code generation context. */ +static bool InitCodeGenContext(JitLlvmCodeGenContext* ctx, GsCodeGen* code_gen, GsCodeGen::LlvmBuilder* builder, + MOT::Table* table, MOT::Index* index, MOT::Table* inner_table = nullptr, MOT::Index* inner_index = nullptr, + int invokeParamCount = 0) +{ + // make sure all members are nullptr for proper cleanup in case of failure + errno_t erc = memset_s(ctx, sizeof(JitLlvmCodeGenContext), 0, sizeof(JitLlvmCodeGenContext)); + securec_check(erc, "\0", "\0"); + + llvm_util::InitLlvmCodeGenContext(ctx, code_gen, builder); + + if (table && !InitTableInfo(&ctx->_table_info, table, index)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "JIT Compile", "Failed to initialize table information for code-generation context"); + DestroyCodeGenContext(ctx); + return false; + } + if (inner_table && inner_index && !InitTableInfo(&ctx->_inner_table_info, inner_table, inner_index)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "JIT Compile", + "Failed to initialize inner-scan table information for code-generation context"); + DestroyCodeGenContext(ctx); + return false; + } + + ctx->m_constCount = 0; + size_t allocSize = sizeof(Const) * MOT_JIT_MAX_CONST; + ctx->m_constValues = (Const*)MOT::MemSessionAlloc(allocSize); + if (ctx->m_constValues == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "JIT Compile", + "Failed to allocate %u bytes for constant array in code-generation context", + allocSize); + DestroyCodeGenContext(ctx); + return false; + } + + // allocate parameter info on global memory, as it will be passed to the global JIT context + if (invokeParamCount > 0) { + allocSize = sizeof(JitParamInfo) * invokeParamCount; + ctx->m_paramInfo = (JitParamInfo*)MOT::MemGlobalAlloc(allocSize); + if (ctx->m_paramInfo == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "JIT Compile", + "Failed to allocate %u bytes for parameter info in code-generation context", + allocSize); + DestroyCodeGenContext(ctx); + return false; + } + erc = memset_s(ctx->m_paramInfo, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + ctx->m_paramCount = invokeParamCount; + } + + InitCodeGenContextTypes(ctx); + InitCodeGenContextFuncs(ctx); + + return true; +} + +/** @brief Initializes a context for compilation. */ +static bool InitCompoundCodeGenContext(JitLlvmCodeGenContext* ctx, GsCodeGen* code_gen, GsCodeGen::LlvmBuilder* builder, + MOT::Table* table, MOT::Index* index, JitCompoundPlan* plan) +{ + // execute normal initialization + if (!InitCodeGenContext(ctx, code_gen, builder, table, index)) { + MOT_REPORT_ERROR( + MOT_ERROR_OOM, "JIT Compile", "Failed to initialize table information for code-generation context"); + return false; + } + + // prepare sub-query table info + ctx->m_subQueryCount = plan->_sub_query_count; + uint32_t allocSize = sizeof(TableInfo) * ctx->m_subQueryCount; + ctx->m_subQueryTableInfo = (TableInfo*)MOT::MemSessionAlloc(allocSize); + if (ctx->m_subQueryTableInfo == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "JIT Compile", + "Failed to allocate %u bytes for %u sub-query table information objects in code-generation context", + allocSize, + (unsigned)ctx->m_subQueryCount); + DestroyCodeGenContext(ctx); + return false; + } + + // initialize sub-query table info + bool result = true; + for (uint32_t i = 0; i < ctx->m_subQueryCount; ++i) { + JitPlan* subPlan = plan->_sub_query_plans[i]; + MOT::Table* subTable = nullptr; + MOT::Index* subIndex = nullptr; + if (subPlan->_plan_type == JIT_PLAN_POINT_QUERY) { + subTable = ((JitSelectPlan*)subPlan)->_query._table; + subIndex = subTable->GetPrimaryIndex(); + } else if (subPlan->_plan_type == JIT_PLAN_RANGE_SCAN) { + subTable = ((JitRangeSelectPlan*)subPlan)->_index_scan._table; + subIndex = ((JitRangeSelectPlan*)subPlan)->_index_scan._index; + } else { + MOT_REPORT_ERROR( + MOT_ERROR_INTERNAL, "JIT Compile", "Invalid sub-plan %u type: %d", i, (int)subPlan->_plan_type); + result = false; + } + if (result && !InitTableInfo(&ctx->m_subQueryTableInfo[i], subTable, subIndex)) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "JIT Compile", + "Failed to initialize sub-query table information for code-generation context"); + result = false; + } + if (!result) { + DestroyCodeGenContext(ctx); + return false; + } + } + + // prepare sub-query data array (for sub-query runtime context) + allocSize = sizeof(JitLlvmCodeGenContext::SubQueryData) * ctx->m_subQueryCount; + ctx->m_subQueryData = (JitLlvmCodeGenContext::SubQueryData*)MOT::MemSessionAlloc(allocSize); + if (ctx->m_subQueryData == nullptr) { + MOT_REPORT_ERROR(MOT_ERROR_OOM, + "JIT Compile", + "Failed to allocate %u bytes for %u sub-query data items in code-generation context", + allocSize, + (unsigned)ctx->m_subQueryCount); + DestroyCodeGenContext(ctx); + return false; + } + errno_t erc = memset_s(ctx->m_subQueryData, allocSize, 0, allocSize); + securec_check(erc, "\0", "\0"); + + return true; +} + +/** @brief Destroys a code generation context. */ +static void DestroyCodeGenContext(JitLlvmCodeGenContext* ctx) +{ + DestroyTableInfo(&ctx->_table_info); + DestroyTableInfo(&ctx->_inner_table_info); + for (uint32_t i = 0; i < ctx->m_subQueryCount; ++i) { + DestroyTableInfo(&ctx->m_subQueryTableInfo[i]); + } + if (ctx->m_subQueryData != nullptr) { + MOT::MemSessionFree(ctx->m_subQueryData); + ctx->m_subQueryData = nullptr; + } + if (ctx->m_subQueryTableInfo != nullptr) { + MOT::MemSessionFree(ctx->m_subQueryTableInfo); + ctx->m_subQueryTableInfo = nullptr; + } + if (ctx->m_constValues != nullptr) { + MOT::MemSessionFree(ctx->m_constValues); + } + if (ctx->m_paramInfo != nullptr) { + MOT::MemGlobalFree(ctx->m_paramInfo); + } + llvm_util::DestroyLlvmCodeGenContext(ctx); +} + +extern int AllocateConstId(JitLlvmCodeGenContext* ctx, int type, Datum value, bool isNull) +{ + int res = -1; + if (ctx->m_constCount == MOT_JIT_MAX_CONST) { + MOT_REPORT_ERROR(MOT_ERROR_RESOURCE_LIMIT, + "JIT Compile", + "Cannot allocate constant identifier, reached limit of %u", + ctx->m_constCount); + } else { + res = ctx->m_constCount++; + ctx->m_constValues[res].consttype = type; + ctx->m_constValues[res].constvalue = value; + ctx->m_constValues[res].constisnull = isNull; + MOT_LOG_TRACE("Allocated constant id: %d", res); + JIT_PRINT_DATUM(MOT::LogLevel::LL_TRACE, "Allocated constant", type, value, isNull); + } + return res; +} + +/** @brief Wraps up an LLVM function (compiles it and prepares a function pointer). */ +static MotJitContext* FinalizeCodegen( + JitLlvmCodeGenContext* ctx, JitCommandType command_type, const char* queryString, JitCodegenStats& codegenStats) +{ + // do GsCodeGen stuff to wrap up + uint64_t startTime = GetSysClock(); + if (!ctx->m_codeGen->verifyFunction(ctx->m_jittedFunction)) { + MOT_LOG_ERROR("Failed to generate jitted code for query: Failed to verify jit function"); +#ifdef LLVM_ENABLE_DUMP + ctx->m_jittedFunction->dump(); +#else + ctx->m_jittedFunction->print(llvm::errs(), nullptr, false, true); +#endif + return nullptr; + } + uint64_t endTime = GetSysClock(); + uint64_t timeMicros = MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime); + codegenStats.m_verifyTime = timeMicros; + MOT_LOG_TRACE("Query '%s' verification time: %" PRIu64 " micros", queryString, timeMicros); + + startTime = GetSysClock(); + ctx->m_codeGen->FinalizeFunction(ctx->m_jittedFunction); + endTime = GetSysClock(); + timeMicros = MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime); + codegenStats.m_finalizeTime = timeMicros; + MOT_LOG_TRACE("Query '%s' finalization time: %" PRIu64 " micros", queryString, timeMicros); + + if (IsMotCodegenPrintEnabled()) { +#ifdef LLVM_ENABLE_DUMP + ctx->m_jittedFunction->dump(); +#else + ctx->m_jittedFunction->print(llvm::errs()); +#endif + } + + // prepare global constant array + JitDatumArray datumArray = {}; + if (ctx->m_constCount > 0) { + if (!PrepareDatumArray(ctx->m_constValues, ctx->m_constCount, &datumArray)) { + MOT_LOG_ERROR("Failed to generate jitted code for query: Failed to prepare constant datum array"); + return nullptr; + } + } + + // that's it, we are ready + JitQueryContext* jit_context = + (JitQueryContext*)AllocJitContext(JIT_CONTEXT_GLOBAL, JitContextType::JIT_CONTEXT_TYPE_QUERY); + if (jit_context == nullptr) { + MOT_LOG_TRACE("Failed to allocate JIT context, aborting code generation"); + return nullptr; + } + + MOT_LOG_DEBUG("Adding function to MCJit"); + ctx->m_codeGen->addFunctionToMCJit(ctx->m_jittedFunction, (void**)&jit_context->m_llvmFunction); + + MOT_LOG_DEBUG("Generating code..."); + startTime = GetSysClock(); + ctx->m_codeGen->enableOptimizations(true); + // this is an important thing + ctx->m_codeGen->compileCurrentModule(false); + endTime = GetSysClock(); + timeMicros = MOT::CpuCyclesLevelTime::CyclesToMicroseconds(endTime - startTime); + codegenStats.m_compileTime = timeMicros; + MOT_LOG_TRACE("Query '%s' compilation time: %" PRIu64 " micros", queryString, timeMicros); + + // setup execution details + jit_context->m_table = ctx->_table_info.m_table; + if (jit_context->m_table != nullptr) { + jit_context->m_tableId = jit_context->m_table->GetTableExId(); + MOT_LOG_TRACE("Installed table id: %" PRIu64, jit_context->m_tableId); + } + jit_context->m_index = ctx->_table_info.m_index; + if (jit_context->m_index != nullptr) { + jit_context->m_indexId = jit_context->m_index->GetExtId(); + MOT_LOG_TRACE("Installed index id: %" PRIu64, jit_context->m_indexId); + } + jit_context->m_codeGen = ctx->m_codeGen; // steal the context + ctx->m_codeGen = nullptr; // prevent destruction + jit_context->m_innerTable = ctx->_inner_table_info.m_table; + if (jit_context->m_innerTable != nullptr) { + jit_context->m_innerTableId = jit_context->m_innerTable->GetTableExId(); + MOT_LOG_TRACE("Installed inner table id: %" PRIu64, jit_context->m_innerTableId); + } + jit_context->m_innerIndex = ctx->_inner_table_info.m_index; + if (jit_context->m_innerIndex != nullptr) { + jit_context->m_innerIndexId = jit_context->m_innerIndex->GetExtId(); + MOT_LOG_TRACE("Installed inner index id: %" PRIu64, jit_context->m_innerIndexId); + } + jit_context->m_aggCount = 0; + jit_context->m_commandType = command_type; + jit_context->m_subQueryCount = 0; + jit_context->m_invokeParamCount = 0; + jit_context->m_invokeParamInfo = nullptr; + jit_context->m_constDatums.m_datumCount = datumArray.m_datumCount; + jit_context->m_constDatums.m_datums = datumArray.m_datums; + jit_context->m_validState = JIT_CONTEXT_VALID; + + return jit_context; +} + +static inline void PrintErrorInfo(const char* queryString) +{ + ErrorData* edata = CopyErrorData(); + MOT_LOG_WARN("Caught exception while generating JIT LLVM code for query '%s': %s", queryString, edata->message); + FlushErrorState(); + FreeErrorData(edata); +} + +#define JIT_TRACE_FAIL(queryType, reason) \ + MOT_LOG_TRACE("Failed to generate jitted code for %s query: %s", queryType, reason) + +static MotJitContext* JitUpdateCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitUpdatePlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT update at thread %p for query: %s", (void*)pthread_self(), query_string); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedUpdate"); + IssueDebugLog("Starting execution of jitted UPDATE"); + + // update is not allowed if we reached soft memory limit + buildIsSoftMemoryLimitReached(ctx); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // build one time filters if required + if (plan->_query.m_oneTimeFilters._filter_count > 0) { + if (!BuildOneTimeFilters(ctx, &plan->_query.m_oneTimeFilters)) { + JIT_TRACE_FAIL("UPDATE", "Failed to build one-time filters"); + return nullptr; + } + } else { + MOT_LOG_TRACE("One-time filters not specified"); + } + + // begin the WHERE clause (this is a point query + if (!buildPointScan(ctx, &plan->_query._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("UPDATE", "Unsupported WHERE clause type"); + return nullptr; + } + + // fetch row for writing + MOT_LOG_DEBUG("Generating update code for point query"); + llvm::Value* row = buildSearchRow(ctx, MOT::AccessType::WR, JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, row, nullptr, &plan->_query._filters, nullptr)) { + JIT_TRACE_FAIL("UPDATE", "Unsupported filter"); + return nullptr; + } + + // prepare a bitmap array + IssueDebugLog("Resetting bitmap set for incremental redo"); + AddResetBitmapSet(ctx); + + // now begin updating columns + IssueDebugLog("Updating row columns"); + if (!writeRowColumns(ctx, row, &plan->_update_exprs, true)) { + JIT_TRACE_FAIL("UPDATE", "Failed to process target entry"); + return nullptr; + } + + // write row + IssueDebugLog("Writing row"); + buildWriteRow(ctx, row, true, nullptr); + + // the next call will be executed only if the previous call to writeRow succeeded + buildIncrementRowsProcessed(ctx); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + return FinalizeCodegen(ctx, JIT_COMMAND_UPDATE, query_string, codegenStats); +} + +static MotJitContext* JitRangeUpdateCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitRangeUpdatePlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT range update at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedRangeUpdate"); + IssueDebugLog("Starting execution of jitted range UPDATE"); + + // update is not allowed if we reached soft memory limit + buildIsSoftMemoryLimitReached(ctx); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // begin the WHERE clause + MOT_LOG_DEBUG("Generating range cursor for range UPDATE query"); + JitLlvmRuntimeCursor cursor = buildRangeCursor(ctx, &plan->_index_scan, JIT_RANGE_SCAN_MAIN, nullptr); + if (cursor.begin_itr == nullptr) { + JIT_TRACE_FAIL("Range UPDATE", "Unsupported WHERE clause type"); + return nullptr; + } + + AddResetBitmapSet(ctx); + + JIT_WHILE_BEGIN(cursor_loop) + llvm::Value* res = AddIsScanEnd(ctx, JIT_INDEX_SCAN_FORWARD, &cursor, JIT_RANGE_SCAN_MAIN); + JIT_WHILE_EVAL_NOT(res); + { + llvm::Value* row = buildGetRowFromIterator(ctx, + JIT_WHILE_COND_BLOCK(), + JIT_WHILE_POST_BLOCK(), + MOT::AccessType::WR, + plan->_index_scan._scan_direction, + &cursor, + JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, row, nullptr, &plan->_index_scan._filters, JIT_WHILE_COND_BLOCK())) { + JIT_TRACE_FAIL("Range UPDATE", "Unsupported filter"); + return nullptr; + } + + // now begin updating columns + IssueDebugLog("Updating row columns"); + if (!writeRowColumns(ctx, row, &plan->_update_exprs, true)) { + JIT_TRACE_FAIL("Range UPDATE", "Failed to process target entry"); + return nullptr; + } + + IssueDebugLog("Writing row"); + buildWriteRow(ctx, row, false, &cursor); + + // the next call will be executed only if the previous call to writeRow succeeded + buildIncrementRowsProcessed(ctx); + + // reset bitmap for next loop + AddResetBitmapSet(ctx); + } + JIT_WHILE_END() + + // cleanup + IssueDebugLog("Reached end of range update loop"); + AddDestroyCursor(ctx, &cursor); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + return FinalizeCodegen(ctx, JIT_COMMAND_RANGE_UPDATE, query_string, codegenStats); +} + +static MotJitContext* JitInsertCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitInsertPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT insert at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedInsert"); + IssueDebugLog("Starting execution of jitted INSERT"); + + // insert is not allowed if we reached soft memory limit + buildIsSoftMemoryLimitReached(ctx); + + // create new row and bitmap set + llvm::Value* row = buildCreateNewRow(ctx); + + // set row null bits + IssueDebugLog("Setting row null bits before insert"); + AddSetRowNullBits(ctx, row); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + IssueDebugLog("Setting row columns"); + if (!writeRowColumns(ctx, row, &plan->_insert_exprs, false)) { + JIT_TRACE_FAIL("INSERT", "Failed to process target entry"); + return nullptr; + } + + IssueDebugLog("Inserting row"); + buildInsertRow(ctx, row); + + // the next call will be executed only if the previous call to writeRow succeeded + buildIncrementRowsProcessed(ctx); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + return FinalizeCodegen(ctx, JIT_COMMAND_INSERT, query_string, codegenStats); +} + +static MotJitContext* JitDeleteCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitDeletePlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT delete at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedDelete"); + IssueDebugLog("Starting execution of jitted DELETE"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // begin the WHERE clause + if (!buildPointScan(ctx, &plan->_query._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("DELETE", "Unsupported WHERE clause type"); + return nullptr; + } + + // fetch row for delete + llvm::Value* row = buildSearchRow(ctx, MOT::AccessType::DEL, JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, row, nullptr, &plan->_query._filters, nullptr)) { + JIT_TRACE_FAIL("DELETE", "Unsupported filter"); + return nullptr; + } + + // row is already cached in concurrency control module, so we do not need to provide an argument + IssueDebugLog("Deleting row"); + buildDeleteRow(ctx); + + // the next call will be executed only if the previous call to deleteRow succeeded + buildIncrementRowsProcessed(ctx); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + return FinalizeCodegen(ctx, JIT_COMMAND_DELETE, query_string, codegenStats); +} + +static MotJitContext* JitRangeDeleteCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitRangeDeletePlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT range delete at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedRangeDelete"); + IssueDebugLog("Starting execution of jitted range DELETE"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // begin the WHERE clause + MOT_LOG_DEBUG("Generating range cursor for range DELETE query"); + JitLlvmRuntimeCursor cursor = buildRangeCursor(ctx, &plan->_index_scan, JIT_RANGE_SCAN_MAIN, nullptr); + if (cursor.begin_itr == nullptr) { + JIT_TRACE_FAIL("Range DELETE", "Unsupported WHERE clause type"); + return nullptr; + } + + JIT_WHILE_BEGIN(cursor_loop) + llvm::Value* res = AddIsScanEnd(ctx, JIT_INDEX_SCAN_FORWARD, &cursor, JIT_RANGE_SCAN_MAIN); + JIT_WHILE_EVAL_NOT(res); + { + llvm::Value* row = buildGetRowFromIterator(ctx, + JIT_WHILE_COND_BLOCK(), + JIT_WHILE_POST_BLOCK(), + MOT::AccessType::DEL, + plan->_index_scan._scan_direction, + &cursor, + JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, row, nullptr, &plan->_index_scan._filters, JIT_WHILE_COND_BLOCK())) { + JIT_TRACE_FAIL("Range DELETE", "Unsupported filter"); + return nullptr; + } + + IssueDebugLog("Deleting row"); + // row is already cached in concurrency control module, so we do not need to provide an argument + buildDeleteRow(ctx); + + // the next call will be executed only if the previous call to deleteRow succeeded + buildIncrementRowsProcessed(ctx); + + // impose limit clause if any + BuildCheckLimitNoState(ctx, plan->_limit_count); + } + JIT_WHILE_END() + + // cleanup + IssueDebugLog("Reached end of range delete loop"); + AddDestroyCursor(ctx, &cursor); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + return FinalizeCodegen(ctx, JIT_COMMAND_RANGE_DELETE, query_string, codegenStats); +} + +static MotJitContext* JitSelectCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitSelectPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT select at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedSelect"); + IssueDebugLog("Starting execution of jitted SELECT"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // build one time filters if required + if (plan->_query.m_oneTimeFilters._filter_count > 0) { + if (!BuildOneTimeFilters(ctx, &plan->_query.m_oneTimeFilters)) { + JIT_TRACE_FAIL("SELECT", "Failed to build one-time filters"); + return nullptr; + } + } else { + MOT_LOG_TRACE("One-time filters not specified"); + } + + // begin the WHERE clause + if (!buildPointScan(ctx, &plan->_query._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("SELECT", "Unsupported WHERE clause type"); + return nullptr; + } + + // fetch row for read + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, row, nullptr, &plan->_query._filters, nullptr)) { + JIT_TRACE_FAIL("SELECT", "Unsupported filter"); + return nullptr; + } + + // now begin selecting columns into result + IssueDebugLog("Selecting columns into result"); + if (!selectRowColumns(ctx, row, &plan->_select_exprs)) { + JIT_TRACE_FAIL("SELECT", "Failed to process target entry"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + + // update number of rows processed + buildIncrementRowsProcessed(ctx); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended (this is a point query) + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_SELECT, query_string, codegenStats); + return jitContext; +} + +static void AddCleanupOldScan(JitLlvmCodeGenContext* ctx) +{ + // emit code to cleanup previous scan in case this is a new scan + JIT_IF_BEGIN(cleanup_old_scan) + JIT_IF_EVAL(ctx->isNewScanValue); + { + IssueDebugLog("Destroying state iterators due to new scan"); + AddDestroyStateIterators(ctx, JIT_RANGE_SCAN_MAIN); + AddDestroyStateIterators(ctx, JIT_RANGE_SCAN_INNER); + AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); + AddResetStateRow(ctx, JIT_RANGE_SCAN_INNER); + // sub-query does not have a stateful execution, so no need to cleanup + } + JIT_IF_END() +} + +/** @brief Generates code for range SELECT query with a possible LIMIT clause. */ +static MotJitContext* JitRangeSelectCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitRangeSelectPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT select at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedRangeSelect"); + IssueDebugLog("Starting execution of jitted range SELECT"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // emit code to cleanup previous scan in case this is a new scan + AddCleanupOldScan(ctx); + + // build one time filters if required, but only if scan is new + bool result = true; + JIT_IF_BEGIN(cleanup_old_scan) + JIT_IF_EVAL(ctx->isNewScanValue); + { + IssueDebugLog("Checking for one-time filters due to new scan"); + if (plan->_index_scan.m_oneTimeFilters._filter_count > 0) { + if (!BuildOneTimeFilters(ctx, &plan->_index_scan.m_oneTimeFilters)) { + JIT_TRACE_FAIL("Range SELECT", "Failed to build one-time filters"); + result = false; + } + } else { + MOT_LOG_TRACE("One-time filters not specified"); + } + } + JIT_IF_END() + if (!result) { + return nullptr; + } + + // prepare stateful scan if not done so already, if no row exists then emit code to return from function + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* row = + buildPrepareStateScanRow(ctx, &plan->_index_scan, JIT_RANGE_SCAN_MAIN, access_mode, nullptr, nullptr, nullptr); + if (row == nullptr) { + JIT_TRACE_FAIL("Range SELECT", "Unsupported WHERE clause type"); + return nullptr; + } + + // select inner and outer row expressions into result tuple (no aggregate because aggregate is not stateful) + IssueDebugLog("Retrieved row from state iterator, beginning to select columns into result tuple"); + if (!selectRowColumns(ctx, row, &plan->_select_exprs)) { + JIT_TRACE_FAIL("Range SELECT", "Failed to select row expressions"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + buildIncrementRowsProcessed(ctx); + + // make sure that next iteration find an empty state row, so scan makes a progress + AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); + + // if a limit clause exists, then increment limit counter and check if reached limit + buildCheckLimit(ctx, plan->_limit_count); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + JitCommandType cmdType = + (plan->_index_scan._scan_type == JIT_INDEX_SCAN_FULL) ? JIT_COMMAND_FULL_SELECT : JIT_COMMAND_RANGE_SELECT; + + MotJitContext* jitContext = FinalizeCodegen(ctx, cmdType, query_string, codegenStats); + + if (jitContext != nullptr) { + if (cmdType == JIT_COMMAND_RANGE_SELECT && plan->m_nonNativeSortParams) { + // This query is using non-native sort. clone the relevant data to context + JitQueryContext* jitQueryContext = (JitQueryContext*)jitContext; + jitQueryContext->m_nonNativeSortParams = + CloneJitNonNativeSortParams(plan->m_nonNativeSortParams, JIT_CONTEXT_GLOBAL); + if (jitQueryContext->m_nonNativeSortParams == nullptr) { + MOT_LOG_ERROR("Failed to clone non native sort data into context."); + DestroyJitContext(jitContext); + jitContext = nullptr; + } + } + } + + return jitContext; +} + +/** @brief Generates code for range SELECT query with aggregator. */ +static MotJitContext* JitAggregateRangeSelectCodegen(JitLlvmCodeGenContext* ctx, const Query* query, + const char* query_string, JitRangeSelectPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT aggregate range select at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedAggregateRangeSelect"); + IssueDebugLog("Starting execution of jitted aggregate range SELECT"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple (we use tuple's resno column as aggregated sum instead of defining local variable) + AddExecClearTuple(ctx); + + // prepare for aggregation + if (!prepareAggregates(ctx, plan->m_aggregates, plan->m_aggCount)) { + JIT_TRACE_FAIL("Aggregate Range SELECT", "Failed to prepare aggregates"); + return nullptr; + } + + // counter for number of aggregates operates on a single row + llvm::Value* aggCount = ctx->m_builder->CreateAlloca(ctx->INT32_T, 0, nullptr, "agg_count"); + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), aggCount, true); + + // pay attention: aggregated range scan is not stateful, since we scan all tuples in one call + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + + // build one time filters if required + llvm::LLVMContext& context = ctx->m_codeGen->context(); + DEFINE_BLOCK(build_agg_select_result, ctx->m_jittedFunction); + if (plan->_index_scan.m_oneTimeFilters._filter_count > 0) { + if (!BuildOneTimeFilters(ctx, &plan->_index_scan.m_oneTimeFilters, build_agg_select_result)) { + JIT_TRACE_FAIL("Aggregate Range SELECT", "Failed to build one-time filters"); + return nullptr; + } + } else { + MOT_LOG_TRACE("One-time filters not specified"); + } + + // begin the WHERE clause + // build range iterators + MOT_LOG_DEBUG("Generating range cursor for range SELECT query"); + JitLlvmRuntimeCursor cursor = buildRangeCursor(ctx, &plan->_index_scan, JIT_RANGE_SCAN_MAIN, nullptr); + if (cursor.begin_itr == nullptr) { + JIT_TRACE_FAIL("Aggregate Range SELECT", "Unsupported WHERE clause type"); + return nullptr; + } + + // when we have select count(*) without any filters we can optimize and skip row copy + bool checkRowExistOnly = false; + if ((plan->m_aggCount == 1) && (plan->m_aggregates[0]._aggreaget_op == JIT_AGGREGATE_COUNT) && + (plan->m_aggregates[0]._table == nullptr) && (plan->_index_scan._filters._filter_count == 0)) { + checkRowExistOnly = true; + } + + llvm::Value* row = nullptr; + JIT_WHILE_BEGIN(cursor_aggregate_loop) + llvm::Value* res = AddIsScanEnd(ctx, plan->_index_scan._scan_direction, &cursor, JIT_RANGE_SCAN_MAIN); + JIT_WHILE_EVAL_NOT(res); + { + if (checkRowExistOnly) { + BuildCheckRowExistsInIterator( + ctx, JIT_WHILE_POST_BLOCK(), plan->_index_scan._scan_direction, &cursor, JIT_RANGE_SCAN_MAIN); + } else { + row = buildGetRowFromIterator(ctx, + JIT_WHILE_COND_BLOCK(), + JIT_WHILE_POST_BLOCK(), + access_mode, + plan->_index_scan._scan_direction, + &cursor, + JIT_RANGE_SCAN_MAIN); + // check for additional filters, if not try to fetch next row + if (!buildFilterRow(ctx, row, nullptr, &plan->_index_scan._filters, JIT_WHILE_COND_BLOCK())) { + JIT_TRACE_FAIL("Aggregate Range SELECT", "Unsupported filter"); + return nullptr; + } + } + + // aggregate into tuple (we use tuple's resno column as aggregated sum instead of defining local variable) + // if row disqualified due to DISTINCT operator then go back to loop test block + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), aggCount); + for (int i = 0; i < plan->m_aggCount; ++i) { + if (!buildAggregateRow(ctx, &plan->m_aggregates[i], i, row, nullptr, aggCount)) { + JIT_TRACE_FAIL("Aggregate Range SELECT", "Unsupported aggregate"); + return nullptr; + } + } + } + JIT_WHILE_END() + + // cleanup + IssueDebugLog("Reached end of aggregate range select loop"); + AddDestroyCursor(ctx, &cursor); + + // wrap up aggregation and write to result tuple + JIT_GOTO(build_agg_select_result); + ctx->m_builder->SetInsertPoint(build_agg_select_result); + buildAggregateResult(ctx, plan->m_aggregates, plan->m_aggCount); + + // store the result tuple + AddExecStoreVirtualTuple(ctx); + + // execute *tp_processed = rows_processed + buildIncrementRowsProcessed(ctx); // aggregate ALWAYS has at least one row processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended (this is an aggregate loop) + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + JitQueryContext* jitContext = + (JitQueryContext*)FinalizeCodegen(ctx, JIT_COMMAND_AGGREGATE_RANGE_SELECT, query_string, codegenStats); + if (jitContext != nullptr) { + jitContext->m_aggCount = plan->m_aggCount; + } + return jitContext; +} + +static bool BuildOneTimeJoinFilters( + JitLlvmCodeGenContext* ctx, JitJoinPlan* plan, const char* planType, llvm::BasicBlock* nextBlock = nullptr) +{ + if (plan->_outer_scan.m_oneTimeFilters._filter_count > 0) { + if (!BuildOneTimeFilters(ctx, &plan->_outer_scan.m_oneTimeFilters, nextBlock)) { + JIT_TRACE_FAIL(planType, "Failed to build outer-scan one-time filters"); + return false; + } + } else { + MOT_LOG_TRACE("One-time filters not specified for outer-scan"); + } + + // build one time filters if required + if (plan->_inner_scan.m_oneTimeFilters._filter_count > 0) { + if (!BuildOneTimeFilters(ctx, &plan->_inner_scan.m_oneTimeFilters, nextBlock)) { + JIT_TRACE_FAIL(planType, "Failed to build inner-scan one-time filters"); + return false; + } + } else { + MOT_LOG_TRACE("One-time filters not specified for inner-scan"); + } + return true; +} + +static MotJitContext* JitPointJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT Point JOIN query at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedPointJoin"); + IssueDebugLog("Starting execution of jitted Point JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Point JOIN")) { + return nullptr; + } + + // search the outer row + if (!buildPointScan(ctx, &plan->_outer_scan._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("Point JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // fetch row for read + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* outer_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, outer_row, nullptr, &plan->_outer_scan._filters, nullptr)) { + JIT_TRACE_FAIL("Point JOIN", "Unsupported outer scan filter"); + return nullptr; + } + + // before we move on to inner point scan, we save the outer row in a safe copy (but for that purpose we need to save + // row in outer scan state) + AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); + AddCopyOuterStateRow(ctx); + + // now search the inner row + if (!buildPointScan(ctx, &plan->_inner_scan._search_exprs, JIT_RANGE_SCAN_INNER, outer_row)) { + JIT_TRACE_FAIL("Point JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + llvm::Value* inner_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_INNER); + + // check for additional filters + if (!buildFilterRow(ctx, outer_row, inner_row, &plan->_inner_scan._filters, nullptr)) { + JIT_TRACE_FAIL("Point JOIN", "Unsupported inner scan filter"); + return nullptr; + } + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // now begin selecting columns into result + MOT_LOG_TRACE("Selecting row columns in the result tuple"); + if (!selectRowColumns(ctx, outer_row_copy, &plan->_select_exprs, inner_row)) { + JIT_TRACE_FAIL("Point JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + + // update number of rows processed + buildIncrementRowsProcessed(ctx); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended (this is a point query) + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_POINT_JOIN, query_string, codegenStats); + return jitContext; +} + +static MotJitContext* JitPointLeftJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT Point LEFT JOIN query at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedPointLeftJoin"); + IssueDebugLog("Starting execution of jitted Point LEFT JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Point Left-JOIN")) { + return nullptr; + } + + // search the outer row + if (!buildPointScan(ctx, &plan->_outer_scan._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("Point Left-JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // fetch row for read + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* outer_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, outer_row, nullptr, &plan->_outer_scan._filters, nullptr)) { + JIT_TRACE_FAIL("Point Left-JOIN", "Unsupported outer scan filter"); + return nullptr; + } + + // before we move on to inner point scan, we save the outer row in a safe copy (but for that purpose we need to save + // row in outer scan state) + AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); + AddCopyOuterStateRow(ctx); + + // now search the inner row + if (!buildPointScan(ctx, &plan->_inner_scan._search_exprs, JIT_RANGE_SCAN_INNER, outer_row)) { + JIT_TRACE_FAIL("Point Left-JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // attention: if the row is null we continue + llvm::Value* filterPassed = ctx->m_builder->CreateAlloca(ctx->INT32_T); + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), filterPassed); + llvm::Value* inner_row = AddSearchRow(ctx, access_mode, JIT_RANGE_SCAN_INNER); + JIT_IF_BEGIN(check_inner_row_found) + JIT_IF_EVAL(inner_row); + { + // check for additional filters, if failed we jump to post block + IssueDebugLog("Inner row found"); + if (!buildFilterRow( + ctx, outer_row_copy, inner_row, &plan->_inner_scan._filters, JIT_IF_CURRENT()->GetPostBlock())) { + JIT_TRACE_FAIL("Point Left-JOIN", "Unsupported inner scan filter"); + return nullptr; + } + // raise flag that all filters passed + IssueDebugLog("Inner row PASSED filter test for LEFT JOIN"); + ctx->m_builder->CreateStore(JIT_CONST_INT32(1), filterPassed); + } + JIT_IF_END() + + // select row columns (if row not found or filter failed, then select null values for inner row) + if (!BuildSelectLeftJoinRowColumns(ctx, outer_row_copy, plan, inner_row, filterPassed)) { + JIT_TRACE_FAIL("Point Left-JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + + // update number of rows processed + buildIncrementRowsProcessed(ctx); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended (this is a point query) + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_POINT_JOIN, query_string, codegenStats); + return jitContext; +} + +/** @brief Generates code for range JOIN query with outer point query and a possible LIMIT clause but without + * aggregation. */ +static MotJitContext* JitOuterPointJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT Outer Point JOIN query at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedOuterPointJoin"); + IssueDebugLog("Starting execution of jitted Outer Point JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // emit code to cleanup previous scan in case this is a new scan + AddCleanupOldScan(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Outer Point JOIN")) { + return nullptr; + } + + // we first check if outer state row was already searched + llvm::Value* outer_row = AddGetStateRow(ctx, JIT_RANGE_SCAN_MAIN); + JIT_IF_BEGIN(check_outer_row_ready) + JIT_IF_EVAL_NOT(outer_row); + { + // search the outer row + if (!buildPointScan(ctx, &plan->_outer_scan._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("Outer Point JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // fetch row for read and check for additional filters + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + outer_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); + if (!buildFilterRow(ctx, outer_row, nullptr, &plan->_outer_scan._filters, nullptr)) { + JIT_TRACE_FAIL("Outer Point JOIN", "Unsupported outer scan filter"); + return nullptr; + } + + // before we move on to inner range scan, we save the outer row in a safe copy (but for that purpose we need to + // save row in outer scan state) + AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); + AddCopyOuterStateRow(ctx); + } + JIT_IF_END() + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // now prepare inner scan if needed, if no row was found then emit code to return from function (since outer scan is + // a point query) + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* inner_row = buildPrepareStateScanRow( + ctx, &plan->_inner_scan, JIT_RANGE_SCAN_INNER, access_mode, outer_row_copy, nullptr, nullptr); + if (inner_row == nullptr) { + JIT_TRACE_FAIL("Outer Point JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + + // now begin selecting columns into result + if (!selectRowColumns(ctx, outer_row_copy, &plan->_select_exprs, inner_row)) { + JIT_TRACE_FAIL("Outer Point JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + + // update number of rows processed + buildIncrementRowsProcessed(ctx); + + // clear inner row for next iteration + AddResetStateRow(ctx, JIT_RANGE_SCAN_INNER); + + // if a limit clause exists, then increment limit counter and check if reached limit + buildCheckLimit(ctx, plan->_limit_count); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_RANGE_JOIN, query_string, codegenStats); + return jitContext; +} + +/** @brief Generates code for range JOIN query with outer point query and a possible LIMIT clause but without + * aggregation. */ +static MotJitContext* JitOuterPointLeftJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, + const char* query_string, JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT Outer Point LEFT JOIN query at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedOuterPointLeftJoin"); + IssueDebugLog("Starting execution of jitted Outer Point LEFT JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // emit code to cleanup previous scan in case this is a new scan + AddCleanupOldScan(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Outer Point Left-JOIN")) { + return nullptr; + } + + // we first check if outer state row was already searched + llvm::Value* outer_row = AddGetStateRow(ctx, JIT_RANGE_SCAN_MAIN); + JIT_IF_BEGIN(check_outer_row_ready) + JIT_IF_EVAL_NOT(outer_row); + { + // search the outer row + if (!buildPointScan(ctx, &plan->_outer_scan._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("Outer Point Left-JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // fetch row for read and check for additional filters + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + outer_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); + if (!buildFilterRow(ctx, outer_row, nullptr, &plan->_outer_scan._filters, nullptr)) { + JIT_TRACE_FAIL("Outer Point Left-JOIN", "Unsupported outer scan filter"); + return nullptr; + } + + // before we move on to inner range scan, we save the outer row in a safe copy (but for that purpose we need to + // save row in outer scan state) + AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); + AddCopyOuterStateRow(ctx); + } + JIT_IF_END() + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // now prepare inner scan if needed, if no row was found or failed to pass filter, then we continue (suppress emit + // return instruction), since in left join we report nulls + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* inner_row = buildPrepareStateScanRow( + ctx, &plan->_inner_scan, JIT_RANGE_SCAN_INNER, access_mode, outer_row_copy, nullptr, nullptr, false); + if (inner_row == nullptr) { + JIT_TRACE_FAIL("Outer Point Left-JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + + // now begin selecting columns into result + if (!selectRowColumns(ctx, outer_row_copy, &plan->_select_exprs, inner_row)) { + JIT_TRACE_FAIL("Outer Point Left-JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + + // update number of rows processed + buildIncrementRowsProcessed(ctx); + + // clear inner row for next iteration + AddResetStateRow(ctx, JIT_RANGE_SCAN_INNER); + + // if a limit clause exists, then increment limit counter and check if reached limit + buildCheckLimit(ctx, plan->_limit_count); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_RANGE_JOIN, query_string, codegenStats); + return jitContext; +} + +/** @brief Generates code for range JOIN query with inner point query and a possible LIMIT clause but without + * aggregation. */ +static MotJitContext* JitInnerPointJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT Inner Point JOIN at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedInnerPointJoin"); + IssueDebugLog("Starting execution of jitted inner point JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // emit code to cleanup previous scan in case this is a new scan + AddCleanupOldScan(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Inner Point JOIN")) { + return nullptr; + } + + // prepare stateful scan if not done so already, if row not found then emit code to return from function (since this + // is an outer scan) + llvm::BasicBlock* fetch_outer_row_bb = nullptr; + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* outer_row = buildPrepareStateScanRow( + ctx, &plan->_outer_scan, JIT_RANGE_SCAN_MAIN, access_mode, nullptr, nullptr, &fetch_outer_row_bb); + if (outer_row == nullptr) { + JIT_TRACE_FAIL("Inner Point JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // before we move on to inner scan, we save the outer row in a safe copy + AddCopyOuterStateRow(ctx); + + // now search the inner row + if (!buildPointScan(ctx, &plan->_inner_scan._search_exprs, JIT_RANGE_SCAN_INNER, outer_row)) { + JIT_TRACE_FAIL("Inner Point JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + llvm::Value* inner_row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_INNER); + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // check for additional filters + if (!buildFilterRow(ctx, outer_row_copy, inner_row, &plan->_inner_scan._filters, fetch_outer_row_bb)) { + JIT_TRACE_FAIL("Inner Point JOIN", "Unsupported inner scan filter"); + return nullptr; + } + + // now begin selecting columns into result + if (!selectRowColumns(ctx, outer_row_copy, &plan->_select_exprs, inner_row)) { + JIT_TRACE_FAIL("Inner Point JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + buildIncrementRowsProcessed(ctx); + + // make sure that next iteration find an empty state row, so scan makes a progress + AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); + + // if a limit clause exists, then increment limit counter and check if reached limit + buildCheckLimit(ctx, plan->_limit_count); + + // generate code for setting output parameter tp_processed value to rows_processed + AddSetTpProcessed(ctx); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_RANGE_JOIN, query_string, codegenStats); + return jitContext; +} + +/** @brief Generates code for range JOIN query with inner point query and a possible LIMIT clause but without + * aggregation. */ +static MotJitContext* JitInnerPointLeftJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, + const char* query_string, JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT Inner Point Left JOIN at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedInnerPointLeftJoin"); + IssueDebugLog("Starting execution of jitted inner point LEFT JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // emit code to cleanup previous scan in case this is a new scan + AddCleanupOldScan(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Inner Point Left-JOIN")) { + return nullptr; + } + + // prepare stateful scan if not done so already, if row not found then emit code to return from function (since this + // is an outer scan) + llvm::BasicBlock* fetch_outer_row_bb = nullptr; + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* outer_row = buildPrepareStateScanRow( + ctx, &plan->_outer_scan, JIT_RANGE_SCAN_MAIN, access_mode, nullptr, nullptr, &fetch_outer_row_bb); + if (outer_row == nullptr) { + JIT_TRACE_FAIL("Inner Point Left-JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // before we move on to inner scan, we save the outer row in a safe copy + AddCopyOuterStateRow(ctx); + + // now search the inner row + if (!buildPointScan(ctx, &plan->_inner_scan._search_exprs, JIT_RANGE_SCAN_INNER, outer_row)) { + JIT_TRACE_FAIL("Inner Point Left-JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // attention: if the row is null we continue + llvm::Value* filterPassed = ctx->m_builder->CreateAlloca(ctx->INT32_T); + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), filterPassed); + llvm::Value* inner_row = AddSearchRow(ctx, access_mode, JIT_RANGE_SCAN_INNER); + JIT_IF_BEGIN(check_inner_row_found) + JIT_IF_EVAL(inner_row); + { + // check for additional filters, if failed we jump to post block + IssueDebugLog("Inner row found"); + if (!buildFilterRow( + ctx, outer_row_copy, inner_row, &plan->_inner_scan._filters, JIT_IF_CURRENT()->GetPostBlock())) { + JIT_TRACE_FAIL("Inner Point Left-JOIN", "Unsupported inner scan filter"); + return nullptr; + } + // raise flag that all filters passed + IssueDebugLog("Inner row PASSED filter test for LEFT JOIN"); + ctx->m_builder->CreateStore(JIT_CONST_INT32(1), filterPassed); + } + JIT_IF_END() + + // select row columns (if row not found or filter failed, then select null values for inner row) + if (!BuildSelectLeftJoinRowColumns(ctx, outer_row_copy, plan, inner_row, filterPassed)) { + JIT_TRACE_FAIL("Inner Point Left-JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + buildIncrementRowsProcessed(ctx); + + // make sure that next iteration find an empty state row, so scan makes a progress + AddResetStateRow(ctx, JIT_RANGE_SCAN_MAIN); + + // if a limit clause exists, then increment limit counter and check if reached limit + buildCheckLimit(ctx, plan->_limit_count); + + // generate code for setting output parameter tp_processed value to rows_processed + AddSetTpProcessed(ctx); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_RANGE_JOIN, query_string, codegenStats); + return jitContext; +} + +/** @brief Generates code for range JOIN query with a possible LIMIT clause but without aggregation. */ +static MotJitContext* JitRangeJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT range JOIN at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedRangeJoin"); + IssueDebugLog("Starting execution of jitted range JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // emit code to cleanup previous scan in case this is a new scan + AddCleanupOldScan(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Range JOIN")) { + return nullptr; + } + + // prepare stateful scan if not done so already + llvm::BasicBlock* fetch_outer_row_bb = nullptr; + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* outer_row = buildPrepareStateScanRow( + ctx, &plan->_outer_scan, JIT_RANGE_SCAN_MAIN, access_mode, nullptr, nullptr, &fetch_outer_row_bb); + if (outer_row == nullptr) { + JIT_TRACE_FAIL("Range JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // before we move on to inner scan, we save the outer row in a safe copy + AddCopyOuterStateRow(ctx); + + // now prepare inner scan if needed + llvm::Value* inner_row = buildPrepareStateScanRow( + ctx, &plan->_inner_scan, JIT_RANGE_SCAN_INNER, access_mode, outer_row, fetch_outer_row_bb, nullptr); + if (inner_row == nullptr) { + JIT_TRACE_FAIL("Range JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // now begin selecting columns into result + if (!selectRowColumns(ctx, outer_row_copy, &plan->_select_exprs, inner_row)) { + JIT_TRACE_FAIL("Range JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + buildIncrementRowsProcessed(ctx); + + // clear inner row for next iteration + AddResetStateRow(ctx, JIT_RANGE_SCAN_INNER); + + // if a limit clause exists, then increment limit counter and check if reached limit + buildCheckLimit(ctx, plan->_limit_count); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_RANGE_JOIN, query_string, codegenStats); + return jitContext; +} + +/** @brief Generates code for range JOIN query with a possible LIMIT clause but without aggregation. */ +static MotJitContext* JitRangeLeftJoinCodegen(JitLlvmCodeGenContext* ctx, const Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT range LEFT JOIN at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedRangeLeftJoin"); + IssueDebugLog("Starting execution of jitted range LEFT JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple even if row is not found later + AddExecClearTuple(ctx); + + // emit code to cleanup previous scan in case this is a new scan + AddCleanupOldScan(ctx); + + // build one time filters if required + if (!BuildOneTimeJoinFilters(ctx, plan, "Range Left-JOIN")) { + return nullptr; + } + + // prepare stateful scan if not done so already + llvm::BasicBlock* fetch_outer_row_bb = nullptr; + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* outer_row = buildPrepareStateScanRow( + ctx, &plan->_outer_scan, JIT_RANGE_SCAN_MAIN, access_mode, nullptr, nullptr, &fetch_outer_row_bb); + if (outer_row == nullptr) { + JIT_TRACE_FAIL("Range Left-JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + // before we move on to inner scan, we save the outer row in a safe copy + AddCopyOuterStateRow(ctx); + + // now prepare inner scan + // attention: if we fail (row not found, or did not pass filters) we still continue and report nulls to user + llvm::Value* inner_row = buildPrepareStateScanRow( + ctx, &plan->_inner_scan, JIT_RANGE_SCAN_INNER, access_mode, outer_row, nullptr, nullptr, false); + if (inner_row == nullptr) { + JIT_TRACE_FAIL("Range Left-JOIN", "Unsupported inner WHERE clause type"); + return nullptr; + } + + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + + // now begin selecting columns into result + if (!selectRowColumns(ctx, outer_row_copy, &plan->_select_exprs, inner_row)) { + JIT_TRACE_FAIL("Range Left-JOIN", "Failed to select row columns into result tuple"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + buildIncrementRowsProcessed(ctx); + + // clear inner row for next iteration + AddResetStateRow(ctx, JIT_RANGE_SCAN_INNER); + + // if a limit clause exists, then increment limit counter and check if reached limit + buildCheckLimit(ctx, plan->_limit_count); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + MotJitContext* jitContext = FinalizeCodegen(ctx, JIT_COMMAND_RANGE_JOIN, query_string, codegenStats); + return jitContext; +} + +/** @brief Generates code for range JOIN query with an aggregator. */ +static MotJitContext* JitAggregateRangeJoinCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for MOT aggregate range JOIN at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedAggregateRangeJoin"); + IssueDebugLog("Starting execution of jitted aggregate range JOIN"); + + // initialize rows_processed local variable + buildResetRowsProcessed(ctx); + + // clear tuple (we use tuple's resno column as aggregated sum instead of defining local variable) + AddExecClearTuple(ctx); + + // prepare for aggregation + if (!prepareAggregates(ctx, plan->m_aggregates, plan->m_aggCount)) { + JIT_TRACE_FAIL("Aggregate Range JOIN", "Failed to prepare aggregates"); + return nullptr; + } + + // counter for number of aggregates operates on a single row + llvm::Value* aggCount = ctx->m_builder->CreateAlloca(ctx->INT32_T, 0, nullptr, "agg_count"); + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), aggCount, true); + + // build one time filters if required + llvm::LLVMContext& context = ctx->m_codeGen->context(); + DEFINE_BLOCK(build_agg_join_result, ctx->m_jittedFunction); + if (!BuildOneTimeJoinFilters(ctx, plan, "Aggregate Range JOIN", build_agg_join_result)) { + return nullptr; + } + + // pay attention: aggregated range scan is not stateful, since we scan all tuples in one call + MOT::AccessType accessMode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + + // begin the WHERE clause + // build range iterators + MOT_LOG_DEBUG("Generating outer loop cursor for range JOIN query"); + JitLlvmRuntimeCursor outer_cursor = buildRangeCursor(ctx, &plan->_outer_scan, JIT_RANGE_SCAN_MAIN, nullptr); + if (outer_cursor.begin_itr == nullptr) { + JIT_TRACE_FAIL("Aggregate Range JOIN", "Unsupported outer WHERE clause type"); + return nullptr; + } + + JIT_WHILE_BEGIN(cursor_aggregate_outer_loop) + llvm::Value* res = AddIsScanEnd(ctx, plan->_outer_scan._scan_direction, &outer_cursor, JIT_RANGE_SCAN_MAIN); + JIT_WHILE_EVAL_NOT(res); + { + llvm::Value* outer_row = buildGetRowFromIterator(ctx, + JIT_WHILE_COND_BLOCK(), + JIT_WHILE_POST_BLOCK(), + accessMode, + plan->_outer_scan._scan_direction, + &outer_cursor, + JIT_RANGE_SCAN_MAIN); + + // check for additional filters, if not try to fetch next row + if (!buildFilterRow(ctx, outer_row, nullptr, &plan->_outer_scan._filters, JIT_WHILE_COND_BLOCK())) { + JIT_TRACE_FAIL("Aggregate Range JOIN", "Unsupported outer scan filter"); + return nullptr; + } + + // before we move on to inner scan, we save the outer row in a safe copy (but for that purpose we need to save + // row in outer scan state) + AddSetStateRow(ctx, outer_row, JIT_RANGE_SCAN_MAIN); + AddCopyOuterStateRow(ctx); + + // now build the inner loop + MOT_LOG_DEBUG("Generating inner loop cursor for range JOIN query"); + JitLlvmRuntimeCursor inner_cursor = buildRangeCursor(ctx, &plan->_inner_scan, JIT_RANGE_SCAN_INNER, outer_row); + if (inner_cursor.begin_itr == nullptr) { + JIT_TRACE_FAIL("Aggregate Range JOIN", "Unsupported outer scan WHERE clause type"); + return nullptr; + } + + JIT_WHILE_BEGIN(cursor_aggregate_inner_loop) + llvm::Value* resInner = + AddIsScanEnd(ctx, plan->_inner_scan._scan_direction, &inner_cursor, JIT_RANGE_SCAN_INNER); + JIT_WHILE_EVAL_NOT(resInner); + { + llvm::Value* inner_row = buildGetRowFromIterator(ctx, + JIT_WHILE_COND_BLOCK(), + JIT_WHILE_POST_BLOCK(), + accessMode, + plan->_inner_scan._scan_direction, + &inner_cursor, + JIT_RANGE_SCAN_INNER); + + // check for additional filters, if not try to fetch next row + if (!buildFilterRow(ctx, nullptr, inner_row, &plan->_inner_scan._filters, JIT_WHILE_COND_BLOCK())) { + JIT_TRACE_FAIL("Aggregate Range JOIN", "Unsupported inner scan filter"); + return nullptr; + } + + // aggregate into tuple (we use tuple's resno column as aggregated sum instead of defining local variable) + // find out to which table the aggregate expression refers, and aggregate it + // if row disqualified due to DISTINCT operator then go back to inner loop test block + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), aggCount, true); + for (int i = 0; i < plan->m_aggCount; ++i) { + bool aggRes = false; + if (plan->m_aggregates[i]._table == ctx->_inner_table_info.m_table) { + aggRes = buildAggregateRow(ctx, &plan->m_aggregates[i], i, nullptr, inner_row, aggCount); + } else { + // retrieve the safe copy of the outer row + llvm::Value* outer_row_copy = AddGetOuterStateRowCopy(ctx); + aggRes = buildAggregateRow(ctx, &plan->m_aggregates[i], i, outer_row_copy, nullptr, aggCount); + } + + if (!aggRes) { + JIT_TRACE_FAIL("Aggregate Range JOIN", "Unsupported aggregate"); + return nullptr; + } + } + } + JIT_WHILE_END() // inner loop + + // cleanup + IssueDebugLog("Reached end of inner loop"); + AddDestroyCursor(ctx, &inner_cursor); + } + JIT_WHILE_END() // outer loop + + // cleanup + IssueDebugLog("Reached end of outer loop"); + AddDestroyCursor(ctx, &outer_cursor); + + // wrap up aggregation and write to result tuple + JIT_GOTO(build_agg_join_result); + ctx->m_builder->SetInsertPoint(build_agg_join_result); + buildAggregateResult(ctx, plan->m_aggregates, plan->m_aggCount); + + // store the result tuple + AddExecStoreVirtualTuple(ctx); + + // execute *tp_processed = rows_processed + buildIncrementRowsProcessed(ctx); // aggregate ALWAYS has at least one row processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), query_string, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + JitQueryContext* jitContext = + (JitQueryContext*)FinalizeCodegen(ctx, JIT_COMMAND_AGGREGATE_JOIN, query_string, codegenStats); + jitContext->m_aggCount = plan->m_aggCount; + return jitContext; +} + +static MotJitContext* JitJoinCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitJoinPlan* plan, JitCodegenStats& codegenStats) +{ + MotJitContext* jit_context = nullptr; + + if (plan->m_aggCount == 0) { + switch (plan->_scan_type) { + case JIT_JOIN_SCAN_POINT: + // special case: this is really a point query + if (plan->_join_type == JitJoinType::JIT_JOIN_INNER) { + jit_context = JitPointJoinCodegen(ctx, query, query_string, plan, codegenStats); + } else if (plan->_join_type == JitJoinType::JIT_JOIN_LEFT) { + jit_context = JitPointLeftJoinCodegen(ctx, query, query_string, plan, codegenStats); + } + break; + + case JIT_JOIN_SCAN_OUTER_POINT: + // special case: outer scan is really a point query + if (plan->_join_type == JitJoinType::JIT_JOIN_INNER) { + jit_context = JitOuterPointJoinCodegen(ctx, query, query_string, plan, codegenStats); + } else if (plan->_join_type == JitJoinType::JIT_JOIN_LEFT) { + jit_context = JitOuterPointLeftJoinCodegen(ctx, query, query_string, plan, codegenStats); + } + break; + + case JIT_JOIN_SCAN_INNER_POINT: + // special case: inner scan is really a point query + if (plan->_join_type == JitJoinType::JIT_JOIN_INNER) { + jit_context = JitInnerPointJoinCodegen(ctx, query, query_string, plan, codegenStats); + } else if (plan->_join_type == JitJoinType::JIT_JOIN_LEFT) { + jit_context = JitInnerPointLeftJoinCodegen(ctx, query, query_string, plan, codegenStats); + } + break; + + case JIT_JOIN_SCAN_RANGE: + if (plan->_join_type == JitJoinType::JIT_JOIN_INNER) { + jit_context = JitRangeJoinCodegen(ctx, query, query_string, plan, codegenStats); + } else if (plan->_join_type == JitJoinType::JIT_JOIN_LEFT) { + jit_context = JitRangeLeftJoinCodegen(ctx, query, query_string, plan, codegenStats); + } + break; + + default: + MOT_LOG_TRACE( + "Cannot generate jitteed code for JOIN plan: Invalid JOIN scan type %d", (int)plan->_scan_type); + break; + } + } else { + jit_context = JitAggregateRangeJoinCodegen(ctx, query, query_string, plan, codegenStats); + } + + return jit_context; +} + +static bool JitSubSelectCodegen(JitLlvmCodeGenContext* ctx, JitCompoundPlan* plan, int subQueryIndex) +{ + MOT_LOG_DEBUG("Generating code for MOT sub-select at thread %p", (intptr_t)pthread_self()); + IssueDebugLog("Executing simple SELECT sub-query"); + + // get the sub-query plan + JitSelectPlan* subPlan = (JitSelectPlan*)plan->_sub_query_plans[subQueryIndex]; + + // begin the WHERE clause + if (!buildPointScan(ctx, &subPlan->_query._search_exprs, JIT_RANGE_SCAN_SUB_QUERY, nullptr, -1, subQueryIndex)) { + JIT_TRACE_FAIL("Compound sub-SELECT", "Unsupported WHERE clause type"); + return false; + } + + // fetch row for read + llvm::Value* row = buildSearchRow(ctx, MOT::AccessType::RD, JIT_RANGE_SCAN_SUB_QUERY, subQueryIndex); + + // check for additional filters + if (!buildFilterRow(ctx, row, nullptr, &subPlan->_query._filters, nullptr)) { + JIT_TRACE_FAIL("Compound sub-SELECT", "Unsupported filter"); + return false; + } + + // now begin selecting columns into result + IssueDebugLog("Selecting column into result"); + if (!selectRowColumns(ctx, row, &subPlan->_select_exprs)) { + JIT_TRACE_FAIL("Compound sub-SELECT", "Failed to process target entry"); + return false; + } + + return true; +} + +/** @brief Generates code for range SELECT sub-query with aggregator. */ +static bool JitSubAggregateRangeSelectCodegen(JitLlvmCodeGenContext* ctx, JitCompoundPlan* plan, int subQueryIndex) +{ + MOT_LOG_DEBUG("Generating code for MOT aggregate range sub-select at thread %p", (intptr_t)pthread_self()); + IssueDebugLog("Executing aggregated range select sub-query"); + + // get the sub-query plan + JitRangeSelectPlan* subPlan = (JitRangeSelectPlan*)plan->_sub_query_plans[subQueryIndex]; + + // prepare for aggregation + if (!prepareAggregates(ctx, subPlan->m_aggregates, subPlan->m_aggCount)) { + JIT_TRACE_FAIL("Aggregate Range sub-SELECT", "Failed to prepare aggregates"); + return false; + } + + // counter for number of aggregates operates on a single row + llvm::Value* aggCount = ctx->m_builder->CreateAlloca(ctx->INT32_T, 0, nullptr, "agg_count"); + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), aggCount, true); + + // build one time filters if required + llvm::LLVMContext& context = ctx->m_codeGen->context(); + DEFINE_BLOCK(build_agg_result, ctx->m_jittedFunction); + if (subPlan->_index_scan.m_oneTimeFilters._filter_count > 0) { + if (!BuildOneTimeFilters(ctx, &subPlan->_index_scan.m_oneTimeFilters, build_agg_result)) { + JIT_TRACE_FAIL("Aggregate Range SELECT", "Failed to build one-time filters"); + return false; + } + } else { + MOT_LOG_TRACE("One-time filters not specified"); + } + + // pay attention: aggregated range scan is not stateful, since we scan all tuples in one call + MOT::AccessType accessMode = MOT::AccessType::RD; + + // begin the WHERE clause + JitIndexScanDirection index_scan_direction = JIT_INDEX_SCAN_FORWARD; + + // build range iterators + MOT_LOG_DEBUG("Generating range cursor for range SELECT sub-query"); + JitLlvmRuntimeCursor cursor = + buildRangeCursor(ctx, &subPlan->_index_scan, JIT_RANGE_SCAN_SUB_QUERY, nullptr, subQueryIndex); + if (cursor.begin_itr == nullptr) { + JIT_TRACE_FAIL("Aggregate Range sub-SELECT", "Unsupported WHERE clause type"); + return false; + } + + JIT_WHILE_BEGIN(cursor_aggregate_loop) + llvm::Value* res = AddIsScanEnd(ctx, index_scan_direction, &cursor, JIT_RANGE_SCAN_SUB_QUERY, subQueryIndex); + JIT_WHILE_EVAL_NOT(res); + { + llvm::Value* row = buildGetRowFromIterator(ctx, + JIT_WHILE_COND_BLOCK(), + JIT_WHILE_POST_BLOCK(), + accessMode, + subPlan->_index_scan._scan_direction, + &cursor, + JIT_RANGE_SCAN_SUB_QUERY, + subQueryIndex); + + // check for additional filters, if not try to fetch next row + if (!buildFilterRow(ctx, row, nullptr, &subPlan->_index_scan._filters, JIT_WHILE_COND_BLOCK())) { + JIT_TRACE_FAIL("Aggregate Range sub-SELECT", "Unsupported filter"); + return false; + } + + // aggregate into tuple (we use tuple's resno column as aggregated sum instead of defining local variable) + // if row disqualified due to DISTINCT operator then go back to loop test block + MOT_ASSERT(subPlan->m_aggCount <= 1); + ctx->m_builder->CreateStore(JIT_CONST_INT32(0), aggCount, true); + for (int i = 0; i < subPlan->m_aggCount; ++i) { + if (!buildAggregateRow(ctx, &subPlan->m_aggregates[i], i, row, nullptr, aggCount)) { + JIT_TRACE_FAIL("Aggregate Range sub-SELECT", "Unsupported aggregate"); + return false; + } + } + } + JIT_WHILE_END() + + // cleanup + IssueDebugLog("Reached end of aggregate range select sub-query loop"); + AddDestroyCursor(ctx, &cursor); + + // wrap up aggregation and write to result tuple (even though this is unfitting to outer query tuple...) + JIT_GOTO(build_agg_result); + ctx->m_builder->SetInsertPoint(build_agg_result); + buildAggregateResult(ctx, subPlan->m_aggregates, subPlan->m_aggCount); + + // coy aggregate result from outer query result tuple into sub-query result tuple + AddCopyAggregateToSubQueryResult(ctx, subQueryIndex); + + return true; +} + +static bool JitSubQueryCodeGen(JitLlvmCodeGenContext* ctx, JitCompoundPlan* plan, int subQueryIndex) +{ + bool result = false; + JitPlan* subPlan = plan->_sub_query_plans[subQueryIndex]; + if (subPlan->_plan_type == JIT_PLAN_POINT_QUERY) { + result = JitSubSelectCodegen(ctx, plan, subQueryIndex); + } else if (subPlan->_plan_type == JIT_PLAN_RANGE_SCAN) { + result = JitSubAggregateRangeSelectCodegen(ctx, plan, subQueryIndex); + } else { + MOT_REPORT_ERROR(MOT_ERROR_INVALID_ARG, + "Generate JIT Code", + "Cannot generate JIT code for sub-query plan: Invalid plan type %d", + (int)subPlan->_plan_type); + } + return result; +} + +static MotJitContext* JitCompoundOuterSelectCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* queryString, + JitSelectPlan* plan, JitCodegenStats& codegenStats) +{ + // begin the WHERE clause + if (!buildPointScan(ctx, &plan->_query._search_exprs, JIT_RANGE_SCAN_MAIN, nullptr)) { + JIT_TRACE_FAIL("Compound Outer-SELECT", "Unsupported WHERE clause type"); + return nullptr; + } + + // fetch row for read + MOT::AccessType access_mode = query->hasForUpdate ? MOT::AccessType::RD_FOR_UPDATE : MOT::AccessType::RD; + llvm::Value* row = buildSearchRow(ctx, access_mode, JIT_RANGE_SCAN_MAIN); + + // check for additional filters + if (!buildFilterRow(ctx, row, nullptr, &plan->_query._filters, nullptr)) { + JIT_TRACE_FAIL("Compound Outer-SELECT", "Unsupported filter"); + return nullptr; + } + + // now begin selecting columns into result + IssueDebugLog("Selecting columns into result"); + if (!selectRowColumns(ctx, row, &plan->_select_exprs)) { + JIT_TRACE_FAIL("Compound Outer-SELECT", "Failed to process target entry"); + return nullptr; + } + + AddExecStoreVirtualTuple(ctx); + + // update number of rows processed + buildIncrementRowsProcessed(ctx); + + // execute *tp_processed = rows_processed + AddSetTpProcessed(ctx); + + // signal to envelope executor scan ended (this is a point query) + AddSetScanEnded(ctx, 1); + + InjectProfileData(ctx, GetActiveNamespace(), queryString, false); + + // return success from calling function + JIT_RETURN(JIT_CONST_INT32((int)MOT::RC_OK)); + + // wrap up + return FinalizeCodegen(ctx, JIT_COMMAND_COMPOUND_SELECT, queryString, codegenStats); +} + +static MotJitContext* JitCompoundOuterCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* queryString, + JitCompoundPlan* plan, JitCodegenStats& codegenStats) +{ + MotJitContext* jitContext = nullptr; + if (plan->_command_type == JIT_COMMAND_SELECT) { + jitContext = JitCompoundOuterSelectCodegen( + ctx, query, queryString, (JitSelectPlan*)plan->_outer_query_plan, codegenStats); + } + // currently other outer query types are not supported + return jitContext; +} + +static MotJitContext* JitCompoundCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitCompoundPlan* plan, JitCodegenStats& codegenStats) +{ + // a compound query plan contains one or more sub-queries that evaluate to a datum that next needs to be fed as a + // parameter to the outer query. We are currently imposing the following limitations: + // 1. one sub-query that can only be a MAX aggregate + // 2. outer query must be a simple point select query. + // + // our main strategy is as follows (based on the fact that each sub-query evaluates into a single value) + // 1. for each sub-query: + // 1.1 execute sub-query and put datum result in sub-query result slot, according to sub-query index + // 2. execute the outer query as a simple query + // 3. whenever we encounter a sub-link expression, it is evaluated as an expression that reads the pre-computed + // sub-query result in step 1.1, according to sub-query index + MOT_LOG_DEBUG("Generating code for MOT compound select at thread %p", (intptr_t)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedCompoundSelect"); + IssueDebugLog("Starting execution of jitted COMPOUND SELECT"); + + // initialize rows_processed local variable (required for sub-queries) + buildResetRowsProcessed(ctx); + + // generate code for sub-query execution + uint32_t subQueryCount = 0; + for (int i = 0; i < plan->_outer_query_plan->_query._search_exprs._count; ++i) { + if (plan->_outer_query_plan->_query._search_exprs._exprs[i]._expr->_expr_type == JIT_EXPR_TYPE_SUBLINK) { + JitSubLinkExpr* subLinkExpr = + (JitSubLinkExpr*)plan->_outer_query_plan->_query._search_exprs._exprs[i]._expr; + if (!JitSubQueryCodeGen(ctx, plan, subLinkExpr->_sub_query_index)) { + JIT_TRACE_FAIL("Compound SELECT", "Failed to generate code for sub-query"); + return nullptr; + } + ++subQueryCount; + } + } + + // clear tuple early, so that we will have a null datum in case outer query finds nothing + AddExecClearTuple(ctx); + + // generate code for the outer query + MotJitContext* jitContext = JitCompoundOuterCodegen(ctx, query, query_string, plan, codegenStats); + if (jitContext == nullptr) { + JIT_TRACE_FAIL("Compound SELECT", "Failed to generate code for outer query"); + return nullptr; + } + + // prepare sub-query data in resulting JIT context (for later execution) + MOT_ASSERT(subQueryCount > 0); + MOT_ASSERT(subQueryCount == plan->_sub_query_count); + if ((subQueryCount > 0) && !PrepareSubQueryData((JitQueryContext*)jitContext, plan)) { + JIT_TRACE_FAIL( + "Compound SELECT", "Failed to prepare tuple table slot array for sub-queries in JIT context object"); + DestroyJitContext(jitContext); + jitContext = nullptr; + } + + return jitContext; +} + +static MotJitContext* JitRangeScanCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitRangeScanPlan* plan, JitCodegenStats& codegenStats) +{ + MotJitContext* jit_context = nullptr; + + switch (plan->_command_type) { + case JIT_COMMAND_UPDATE: + jit_context = JitRangeUpdateCodegen(ctx, query, query_string, (JitRangeUpdatePlan*)plan, codegenStats); + break; + + case JIT_COMMAND_SELECT: { + JitRangeSelectPlan* range_select_plan = (JitRangeSelectPlan*)plan; + if (range_select_plan->m_aggCount == 0) { + jit_context = JitRangeSelectCodegen(ctx, query, query_string, range_select_plan, codegenStats); + } else { + jit_context = JitAggregateRangeSelectCodegen(ctx, query, query_string, range_select_plan, codegenStats); + } + } break; + + case JIT_COMMAND_DELETE: + jit_context = JitRangeDeleteCodegen(ctx, query, query_string, (JitRangeDeletePlan*)plan, codegenStats); + break; + + default: + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Generate JIT Code", + "Invalid point query JIT plan command type %d", + (int)plan->_command_type); + break; + } + + return jit_context; +} + +static MotJitContext* JitPointQueryCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* query_string, + JitPointQueryPlan* plan, JitCodegenStats& codegenStats) +{ + MotJitContext* jitContext = nullptr; + switch (plan->_command_type) { + case JIT_COMMAND_UPDATE: + jitContext = JitUpdateCodegen(ctx, query, query_string, (JitUpdatePlan*)plan, codegenStats); + break; + + case JIT_COMMAND_DELETE: + jitContext = JitDeleteCodegen(ctx, query, query_string, (JitDeletePlan*)plan, codegenStats); + break; + + case JIT_COMMAND_SELECT: + jitContext = JitSelectCodegen(ctx, query, query_string, (JitSelectPlan*)plan, codegenStats); + break; + + default: + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Generate JIT Code", + "Invalid point query JIT plan command type %d", + (int)plan->_command_type); + break; + } + return jitContext; +} + +static MotJitContext* JitInvokeCodegen(JitLlvmCodeGenContext* ctx, Query* query, const char* queryString, + JitInvokePlan* plan, JitCodegenStats& codegenStats) +{ + MOT_LOG_DEBUG("Generating code for invoke at thread %p", (void*)pthread_self()); + + // prepare the jitted function (declare, get arguments into context and define locals) + CreateJittedFunction(ctx, "MotJittedInvoke"); + IssueDebugLog("Starting execution of jitted INVOKE"); + + IssueDebugLog("Preparing function call arguments"); + MOT_LOG_TRACE("Preparing %d function call arguments", plan->_arg_count); + + // prepare parameter list info + llvm::Value* paramListInfo = AddGetInvokeParamListInfo(ctx); + MOT_LOG_TRACE("Preparing %d plan arguments", plan->_arg_count); + for (int i = 0; i < plan->_arg_count; ++i) { + if (plan->_args[i]->_expr_type == JIT_EXPR_TYPE_PARAM) { + ctx->m_paramInfo[i].m_mode = JitParamMode::JIT_PARAM_DIRECT; + ctx->m_paramInfo[i].m_index = ((JitParamExpr*)plan->_args[i])->_param_id; + continue; + } + ctx->m_paramInfo[i].m_mode = JitParamMode::JIT_PARAM_COPY; + ctx->m_paramInfo[i].m_index = -1; + llvm::Value* expr = ProcessExpr(ctx, nullptr, nullptr, plan->_args[i]); + if (expr == nullptr) { + JIT_TRACE_FAIL("INVOKE", "Failed to process parameter expression"); + return nullptr; + } + llvm::Value* exprIsNull = AddGetExprIsNull(ctx); + AddSetParamValue(ctx, paramListInfo, JIT_CONST_INT32(i), plan->_args[i]->_result_type, expr, exprIsNull); + } + + // pass default parameters if needed + uint64_t paramCount = plan->m_functionPlan ? plan->m_functionPlan->m_argCount : get_func_nargs(plan->_function_id); + if (plan->m_defaultParamCount > 0) { + for (int i = 0; i < plan->m_defaultParamCount; ++i) { + llvm::Value* expr = ProcessExpr(ctx, nullptr, nullptr, plan->m_defaultParams[i]); + if (expr == nullptr) { + JIT_TRACE_FAIL("INVOKE", "Failed to process default value expression"); + return nullptr; + } + llvm::Value* exprIsNull = AddGetExprIsNull(ctx); + int paramPos = plan->_arg_count + i; + Oid resultType = plan->m_defaultParams[i]->_result_type; + AddSetParamValue(ctx, paramListInfo, JIT_CONST_INT32(paramPos), resultType, expr, exprIsNull); + ctx->m_paramInfo[paramPos].m_mode = JitParamMode::JIT_PARAM_COPY; + ctx->m_paramInfo[paramPos].m_index = -1; + } + } + + InjectInvokeProfileData(ctx, GetActiveNamespace(), queryString, true); + // invoke the stored procedure + llvm::Value* retValue = AddInvokeStoredProcedure(ctx); + InjectInvokeProfileData(ctx, GetActiveNamespace(), queryString, false); + + // NOTE: in case of SP1 calling SP2, where SP2 threw unhandled exception, then there is no need here to throw + // exception, because the sql code is pulled up into JIT context of SP1, where it is handled there + + InjectProfileData(ctx, GetActiveNamespace(), queryString, false); + + // return success from calling function + JIT_RETURN(retValue); + + // wrap up + JitQueryContext* jitContext = (JitQueryContext*)FinalizeCodegen(ctx, JIT_COMMAND_INVOKE, queryString, codegenStats); + if (jitContext != nullptr) { + // setup parameter info (for later preparing) - as much as the called function needs + MOT_LOG_TRACE("Installing %d parameters in source context (%d used)", paramCount, plan->_arg_count); + MOT_ASSERT((paramCount == 0 && ctx->m_paramInfo == nullptr) || (paramCount > 0 && ctx->m_paramInfo != nullptr)); + jitContext->m_invokeParamCount = paramCount; + jitContext->m_invokeParamInfo = ctx->m_paramInfo; + ctx->m_paramInfo = nullptr; + // compile the called stored procedure if there is such + if (plan->m_functionPlan != nullptr) { + jitContext->m_invokedQueryString = nullptr; + jitContext->m_invokedFunctionOid = InvalidOid; + jitContext->m_invokedFunctionTxnId = InvalidTransactionId; + jitContext->m_invokeContext = + (JitFunctionContext*)ProcessInvokedPlan(plan->m_functionPlan, JIT_CONTEXT_GLOBAL_SECONDARY); + if (jitContext->m_invokeContext == nullptr) { + MOT_LOG_TRACE("Failed to generate code for invoked stored procedure: %s", plan->_function_name); + DestroyJitContext(jitContext); + return nullptr; + } + jitContext->m_invokeContext->m_parentContext = jitContext; + } else { + jitContext->m_invokedQueryString = MakeInvokedQueryString(plan->_function_name, plan->_function_id); + if (jitContext->m_invokedQueryString == nullptr) { + MOT_LOG_TRACE("Failed to prepare invoked query string for stored procedure: %s", plan->_function_name); + DestroyJitContext(jitContext); + return nullptr; + } + jitContext->m_invokedFunctionOid = plan->_function_id; + jitContext->m_invokedFunctionTxnId = plan->m_functionTxnId; + jitContext->m_invokeContext = nullptr; + } + } + + return jitContext; +} + +static bool InitCodeGenContextByPlan( + JitLlvmCodeGenContext* ctx, GsCodeGen* codeGen, GsCodeGen::LlvmBuilder* builder, JitPlan* plan) +{ + // ATTENTION: in case of failure the code-gen object is destroyed, otherwise it is owned by the context + bool result = false; + switch (plan->_plan_type) { + case JIT_PLAN_INSERT_QUERY: { + JitInsertPlan* insertPlan = (JitInsertPlan*)plan; + MOT::Table* table = insertPlan->_table; + result = InitCodeGenContext(ctx, codeGen, builder, table, table->GetPrimaryIndex()); + break; + } + + case JIT_PLAN_POINT_QUERY: { + JitPointQueryPlan* pqueryPlan = (JitPointQueryPlan*)plan; + MOT::Table* table = pqueryPlan->_query._table; + result = InitCodeGenContext(ctx, codeGen, builder, table, table->GetPrimaryIndex()); + break; + } + + case JIT_PLAN_RANGE_SCAN: { + JitRangeScanPlan* rscanPlan = (JitRangeScanPlan*)plan; + MOT::Table* table = rscanPlan->_index_scan._table; + MOT::Index* index = rscanPlan->_index_scan._index; + result = InitCodeGenContext(ctx, codeGen, builder, table, index); + break; + } + + case JIT_PLAN_JOIN: { + JitJoinPlan* joinPlan = (JitJoinPlan*)plan; + MOT::Table* outerTable = joinPlan->_outer_scan._table; + MOT::Index* outerIndex = joinPlan->_outer_scan._index; + MOT::Table* innerTable = joinPlan->_inner_scan._table; + MOT::Index* innerIndex = joinPlan->_inner_scan._index; + result = InitCodeGenContext(ctx, codeGen, builder, outerTable, outerIndex, innerTable, innerIndex); + break; + } + + case JIT_PLAN_COMPOUND: { + JitCompoundPlan* compoundPlan = (JitCompoundPlan*)plan; + MOT::Table* table = compoundPlan->_outer_query_plan->_query._table; + result = InitCompoundCodeGenContext(ctx, codeGen, builder, table, table->GetPrimaryIndex(), compoundPlan); + break; + } + + case JIT_PLAN_INVOKE: { + JitInvokePlan* invokePlan = (JitInvokePlan*)plan; + uint64_t paramCount = invokePlan->m_functionPlan ? invokePlan->m_functionPlan->m_argCount + : get_func_nargs(invokePlan->_function_id); + result = InitCodeGenContext(ctx, codeGen, builder, nullptr, nullptr, nullptr, nullptr, (int)paramCount); + break; + } + + default: + MOT_REPORT_ERROR( + MOT_ERROR_INTERNAL, "Generate JIT Code", "Invalid JIT plan type %d", (int)plan->_plan_type); + FreeGsCodeGen(codeGen); + break; + } + + return result; +} +/* function name:JitCodegenLlvmQuery + function purpose:Generate LLVM JIT compiled code for the given query + input:query pointer,query string ,Pointer to jit plan + output:Pointer to jit context + note:none + annotator:liushifa + annotate time:2023/09/21 11:52:32 + contact:3325287047@qq.com +*/ +MotJitContext* JitCodegenLlvmQuery(Query* query, const char* query_string, JitPlan* plan, JitCodegenStats& codegenStats) +{ + // Ensure that LLVM's code generation tool is valid + JIT_ASSERT_LLVM_CODEGEN_UTIL_VALID(); + GsCodeGen* codeGen = SetupCodegenEnv(); + if (codeGen == nullptr) { + return nullptr; + } + //Create an LLVM generator + GsCodeGen::LlvmBuilder builder(codeGen->context()); + //Initialize code generation context + volatile JitLlvmCodeGenContext ctx = {}; + if (!InitCodeGenContextByPlan((JitLlvmCodeGenContext*)&ctx, codeGen, &builder, plan)) { + // ATTENTION: in case of error code-gen object is destroyed already + MOT_LOG_TRACE("Failed to initialize code-gen context by plan"); + return nullptr; + } + ctx.m_queryString = query_string; + + // NOTE: every variable used after catch needs to be volatile (see longjmp() man page) + volatile MotJitContext* jitContext = nullptr; + volatile MemoryContext origCxt = CurrentMemoryContext; + MOT_LOG_DEBUG("*** Attempting to generate planned LLVM-jitted code for query: %s", query_string); + // Attempt to generate JIT compiled code for different types of queries + PG_TRY(); + { + switch (plan->_plan_type) { + case JIT_PLAN_INSERT_QUERY: + jitContext = JitInsertCodegen( + (JitLlvmCodeGenContext*)&ctx, query, query_string, (JitInsertPlan*)plan, codegenStats); + break; + + case JIT_PLAN_POINT_QUERY: + jitContext = JitPointQueryCodegen( + (JitLlvmCodeGenContext*)&ctx, query, query_string, (JitPointQueryPlan*)plan, codegenStats); + break; + + case JIT_PLAN_RANGE_SCAN: + jitContext = JitRangeScanCodegen( + (JitLlvmCodeGenContext*)&ctx, query, query_string, (JitRangeScanPlan*)plan, codegenStats); + break; + + case JIT_PLAN_JOIN: + jitContext = + JitJoinCodegen((JitLlvmCodeGenContext*)&ctx, query, query_string, (JitJoinPlan*)plan, codegenStats); + break; + + case JIT_PLAN_COMPOUND: + jitContext = JitCompoundCodegen( + (JitLlvmCodeGenContext*)&ctx, query, query_string, (JitCompoundPlan*)plan, codegenStats); + break; + + case JIT_PLAN_INVOKE: + jitContext = JitInvokeCodegen( + (JitLlvmCodeGenContext*)&ctx, query, query_string, (JitInvokePlan*)plan, codegenStats); + break; + + default: + MOT_REPORT_ERROR( + MOT_ERROR_INTERNAL, "Generate JIT Code", "Invalid JIT plan type %d", (int)plan->_plan_type); + break; + } + } + PG_CATCH(); + { + (void)MemoryContextSwitchTo(origCxt); + PrintErrorInfo(query_string); + } + PG_END_TRY(); + + // cleanup + DestroyCodeGenContext((JitLlvmCodeGenContext*)&ctx); + + if (jitContext == nullptr) { + MOT_LOG_TRACE("Failed to generate LLVM-jitted code for query: %s", query_string); + } else { + MOT_LOG_DEBUG( + "Got LLVM-jitted function %p after compile, for query: %s", jitContext->m_llvmFunction, query_string); + } + + // reset compile state for robustness + llvm_util::JitResetCompileState(); + JIT_ASSERT_LLVM_CODEGEN_UTIL_VALID(); + return (MotJitContext*)jitContext; +} + +/* function name:JitExecLlvmQuery + function purpose:Execute a specific llvm generated query. + input: + @param jitContext A jit context containing query-related information. + @param params Query parameters. + @param slot Tuple table slot used to temporarily store query results. + @param tuplesProcessed Counter of the number of tuples processed. + @param scanEnded Flag indicating whether the query scan has ended. + @param newScan Flag indicating whether to start a new scan. + output:Returns the execution result status of the query. + note:none + annotator:liushifa + annotate time:2023/09/22 20:54:33 + contact:3325287047@qq.com +*/ +extern int JitExecLlvmQuery(JitQueryContext* jitContext, ParamListInfo params, TupleTableSlot* slot, + uint64_t* tuplesProcessed, int* scanEnded, int newScan) +{ + // Make sure the context type provided is the context used for the query. + MOT_ASSERT(jitContext->m_contextType == JitContextType::JIT_CONTEXT_TYPE_QUERY); + JitQueryExecState* execState = (JitQueryExecState*)jitContext->m_execState; + + int result = 0; + // Record debugging information when query execution begins. + MOT_LOG_TRACE("Calling sigsetjmp on faultBuf %p, query: %s", execState->m_faultBuf, jitContext->m_queryString); + + // we setup a jump buffer in the execution state for fault handling + if (sigsetjmp(execState->m_faultBuf, 1) == 0) { + // execute the jitted-function + // If there are no runtime errors, execute the jit function generated by llvm + result = jitContext->m_llvmFunction(jitContext->m_table, + jitContext->m_index, + execState->m_searchKey, + execState->m_bitmapSet, + params, + slot, + tuplesProcessed, + scanEnded, + newScan, + execState->m_endIteratorKey, + jitContext->m_innerTable, + jitContext->m_innerIndex, + execState->m_innerSearchKey, + execState->m_innerEndIteratorKey); + } else { + uint64_t faultCode = execState->m_exceptionValue; + MOT_REPORT_ERROR(MOT_ERROR_INTERNAL, + "Execute JIT", + "Encountered run-time fault %" PRIu64 " (%s), exception status: %" PRIu64 ", exception value: %" PRIu64, + faultCode, + llvm_util::LlvmRuntimeFaultToString(faultCode), + execState->m_exceptionStatus, + execState->m_exceptionValue); + result = MOT::RC_JIT_SP_EXCEPTION; + } + + MOT_LOG_TRACE("JitExecLlvmQuery returned %d/%u for query: %s", result, jitContext->m_rc, jitContext->m_queryString); + return result; +} +} // namespace JitExec