natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Safeguard Full repair against disk protection 

As per CASSANDRA-20045, we want to prevent full repair against
disk full scenarios. Current protection exists only for incremental
repair. This change updates the config name to not be
incremental repair specific, using the Replace annotation.

patch by Himanshu Jindal; reviewed by David Capwell, Jaydeepkumar Chovatia for CASSANDRA-20045
This commit is contained in:
Himanshu Jindal 2025-08-12 09:59:36 -07:00 committed by Jaydeepkumar Chovatia
parent 12dcb130f7
commit daef7b5b98
9 changed files with 31 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
5.1
* AutoRepair: Safeguard Full repair against disk protection(CASSANDRA-20045)
* BEGIN TRANSACTION crashes if a mutation touches multiple rows (CASSANDRA-20844)
* Fix version range check in MessagingService.getVersionOrdinal (CASSANDRA-20842)
* Allow custom constraints to be loaded via SPI (CASSANDRA-20824)

8
conf/cassandra.yaml
View File

@ -2683,10 +2683,14 @@ storage_compatibility_mode: NONE
# the given value. Defaults to disabled.
# reject_repair_compaction_threshold: 1024
# At least 20% of disk must be unused to run incremental repair. It is useful to avoid disks filling up during
# incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily.
# At least 20% of disk must be unused to run repair. It is useful to avoid disks filling up during
# repair as anti-compaction during repair may contribute to additional space temporarily.
# if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason)
# then set the ratio to 0.0
# repair_disk_headroom_reject_ratio: 0.2;
# This is the deprecated config which was used to safeguard incremental repairs. Use repair_disk_headroom_reject_ratio
# instead as it safeguards against all repairs.
# incremental_repair_disk_headroom_reject_ratio: 0.2;
# Configuration for Auto Repair Scheduler.

6
conf/cassandra_latest.yaml
View File

@ -2374,11 +2374,11 @@ storage_compatibility_mode: NONE
# the given value. Defaults to disabled.
# reject_repair_compaction_threshold: 1024
# At least 20% of disk must be unused to run incremental repair. It is useful to avoid disks filling up during
# incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily.
# At least 20% of disk must be unused to run repair. It is useful to avoid disks filling up during
# repair as anti-compaction during repair may contribute to additional space temporarily.
# if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason)
# then set the ratio to 0.0
# incremental_repair_disk_headroom_reject_ratio: 0.2;
# repair_disk_headroom_reject_ratio: 0.2;
# Configuration for Auto Repair Scheduler.
#

4
doc/modules/cassandra/pages/managing/operating/auto_repair.adoc
View File

@ -318,9 +318,9 @@ When enabling auto_repair, it is advisable to configure the top level `reject_re
configuration in cassandra.yaml as a backpressure mechanism to reject new repairs on instances that have many
pending compactions.
==== Tune `incremental_repair_disk_headroom_reject_ratio`
==== Tune `repair_disk_headroom_reject_ratio`
By default, incremental repairs will be rejected if less than 20% of disk is available. If one wishes to be
By default, repairs will be rejected if less than 20% of disk is available. If one wishes to be
conservative this top level configuration could be increased to a larger value to prevent filling your data directories.
== Table configuration

5
src/java/org/apache/cassandra/config/Config.java
View File

@ -370,9 +370,10 @@ public class Config
// The number of executors to use for building secondary indexes
public volatile int concurrent_index_builders = 2;
// at least 20% of disk must be unused to run incremental repair
// at least 20% of disk must be unused to run repair
// if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) then set the ratio to 0.0
public volatile double incremental_repair_disk_headroom_reject_ratio = 0.2;
@Replaces(oldName = "incremental_repair_disk_headroom_reject_ratio")
public volatile double repair_disk_headroom_reject_ratio = 0.2;
/**
* @deprecated retry support removed on CASSANDRA-10992

10
src/java/org/apache/cassandra/config/DatabaseDescriptor.java
View File

@ -5955,18 +5955,18 @@ public class DatabaseDescriptor
return conf.auto_repair;
}
public static double getIncrementalRepairDiskHeadroomRejectRatio()
public static double getRepairDiskHeadroomRejectRatio()
{
return conf.incremental_repair_disk_headroom_reject_ratio;
return conf.repair_disk_headroom_reject_ratio;
}
public static void setIncrementalRepairDiskHeadroomRejectRatio(double value)
public static void setRepairDiskHeadroomRejectRatio(double value)
{
if (value < 0.0 || value > 1.0)
{
throw new IllegalArgumentException("Value must be >= 0 and <= 1 for incremental_repair_disk_headroom_reject_ratio");
throw new IllegalArgumentException("Value must be >= 0 and <= 1 for repair_disk_headroom_reject_ratio");
}
conf.incremental_repair_disk_headroom_reject_ratio = value;
conf.repair_disk_headroom_reject_ratio = value;
}
@VisibleForTesting

2
src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java
View File

@ -123,7 +123,7 @@ public class RepairMessageVerbHandler implements IVerbHandler<RepairMessage>
sendFailureResponse(message);
return;
}
if (!ActiveRepairService.verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind, prepareMessage.isIncremental))
if (!ActiveRepairService.verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind))
{
// error is logged in verifyDiskHeadroomThreshold
state.phase.fail("Not enough disk headroom to perform incremental repair");

11
src/java/org/apache/cassandra/service/ActiveRepairService.java
View File

@ -670,7 +670,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
public Future<?> prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set<InetAddressAndPort> endpoints, RepairOption options, boolean isForcedRepair, List<ColumnFamilyStore> columnFamilyStores)
{
if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind(), options.isIncremental()))
if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind()))
failRepair(parentRepairSession, "Rejecting incoming repair, disk usage above threshold"); // failRepair throws exception
if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind()))
@ -730,11 +730,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
return promise;
}
public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind, boolean isIncremental)
public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind)
{
if (!isIncremental) // disk headroom is required for anti-compaction which is only performed by incremental repair
return true;
double diskUsage = DiskUsageMonitor.instance.getDiskUsage();
double rejectRatio = ActiveRepairService.instance().getIncrementalRepairDiskHeadroomRejectRatio();
@ -1110,12 +1107,12 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai
public double getIncrementalRepairDiskHeadroomRejectRatio()
{
return DatabaseDescriptor.getIncrementalRepairDiskHeadroomRejectRatio();
return DatabaseDescriptor.getRepairDiskHeadroomRejectRatio();
}
public void setIncrementalRepairDiskHeadroomRejectRatio(double value)
{
DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(value);
DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(value);
}
/**

12
test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java
View File

@ -530,9 +530,9 @@ public class ActiveRepairServiceTest
}
}
public void testVerifyDiskHeadroomThresholdFullRepair()
public void testVerifyDefaultDiskHeadroomThreshold()
{
Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, false));
Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE));
}
@Test
@ -540,9 +540,9 @@ public class ActiveRepairServiceTest
{
DiskUsageMonitor.instance = diskUsageMonitor;
when(diskUsageMonitor.getDiskUsage()).thenReturn(1.0);
DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(1.0);
DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(1.0);
Assert.assertFalse(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true));
Assert.assertFalse(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE));
}
@Test
@ -550,9 +550,9 @@ public class ActiveRepairServiceTest
{
DiskUsageMonitor.instance = diskUsageMonitor;
when(diskUsageMonitor.getDiskUsage()).thenReturn(0.0);
DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(0.0);
DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(0.0);
Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true));
Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE));
}
@Test(expected = RuntimeException.class)