From 7f1503d9c9b78512a34c38c0df98b4bd89538d09 Mon Sep 17 00:00:00 2001
From: Matt Byrd <matthew_byrd@apple.com>
Date: Tue, 18 Mar 2025 11:04:00 -0700
Subject: [PATCH] Split out truncation record locking to prevent it being
 blocked by slow interval tree build on removeEndpoint

Patch by Matt Byrd; reviewed by Ariel Weisberg, Marcus Eriksson for CASSANDRA-20480
---
 CHANGES.txt                                   |  1 +
 .../apache/cassandra/db/SystemKeyspace.java   | 45 ++++++++++++-------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/CHANGES.txt b/CHANGES.txt
index c03ffcafa4..45a4f16b56 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 5.1
+ * Split out truncation record lock (CASSANDRA-20480)
  * Throw new IndexBuildInProgressException when queries fail during index build, instead of IndexNotAvailableException (CASSANDRA-20402)
  * Fix Paxos repair interrupts running transactions (CASSANDRA-20469)
  * Various fixes in constraint framework (CASSANDRA-20481)
diff --git a/src/java/org/apache/cassandra/db/SystemKeyspace.java b/src/java/org/apache/cassandra/db/SystemKeyspace.java
index 64436a9403..81186c512a 100644
--- a/src/java/org/apache/cassandra/db/SystemKeyspace.java
+++ b/src/java/org/apache/cassandra/db/SystemKeyspace.java
@@ -606,6 +606,8 @@ public final class SystemKeyspace
 
     private static volatile Map<TableId, Pair<CommitLogPosition, Long>> truncationRecords;
 
+    private static final Object truncationRecordLock = new Object();
+
     public enum BootstrapState
     {
         NEEDS_BOOTSTRAP,
@@ -801,27 +803,33 @@ public final class SystemKeyspace
         return status;
     }
 
-    public static synchronized void saveTruncationRecord(ColumnFamilyStore cfs, long truncatedAt, CommitLogPosition position)
+    public static void saveTruncationRecord(ColumnFamilyStore cfs, long truncatedAt, CommitLogPosition position)
     {
-        String req = "UPDATE system.%s SET truncated_at = truncated_at + ? WHERE key = '%s'";
-        executeInternal(format(req, LOCAL, LOCAL), truncationAsMapEntry(cfs, truncatedAt, position));
-        truncationRecords = null;
-        forceBlockingFlush(LOCAL);
+        synchronized (truncationRecordLock)
+        {
+            String req = "UPDATE system.%s SET truncated_at = truncated_at + ? WHERE key = '%s'";
+            executeInternal(format(req, LOCAL, LOCAL), truncationAsMapEntry(cfs, truncatedAt, position));
+            truncationRecords = null;
+            forceBlockingFlush(LOCAL);
+        }
     }
 
     /**
      * This method is used to remove information about truncation time for specified column family
      */
-    public static synchronized void removeTruncationRecord(TableId id)
+    public static void removeTruncationRecord(TableId id)
     {
-        Pair<CommitLogPosition, Long> truncationRecord = getTruncationRecord(id);
-        if (truncationRecord == null)
-            return;
+        synchronized (truncationRecordLock)
+        {
+            Pair<CommitLogPosition, Long> truncationRecord = getTruncationRecord(id);
+            if (truncationRecord == null)
+                return;
 
-        String req = "DELETE truncated_at[?] from system.%s WHERE key = '%s'";
-        executeInternal(format(req, LOCAL, LOCAL), id.asUUID());
-        truncationRecords = null;
-        forceBlockingFlush(LOCAL);
+            String req = "DELETE truncated_at[?] from system.%s WHERE key = '%s'";
+            executeInternal(format(req, LOCAL, LOCAL), id.asUUID());
+            truncationRecords = null;
+            forceBlockingFlush(LOCAL);
+        }
     }
 
     private static Map<UUID, ByteBuffer> truncationAsMapEntry(ColumnFamilyStore cfs, long truncatedAt, CommitLogPosition position)
@@ -850,11 +858,14 @@ public final class SystemKeyspace
         return record == null ? Long.MIN_VALUE : record.right;
     }
 
-    private static synchronized Pair<CommitLogPosition, Long> getTruncationRecord(TableId id)
+    private static Pair<CommitLogPosition, Long> getTruncationRecord(TableId id)
     {
-        if (truncationRecords == null)
-            truncationRecords = readTruncationRecords();
-        return truncationRecords.get(id);
+        synchronized (truncationRecordLock)
+        {
+            if (truncationRecords == null)
+                truncationRecords = readTruncationRecords();
+            return truncationRecords.get(id);
+        }
     }
 
     private static Map<TableId, Pair<CommitLogPosition, Long>> readTruncationRecords()