diff --git a/CHANGES.txt b/CHANGES.txt index b7eb1c8b49..f45d70e83c 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.1 + * AutoRepair: Safeguard Full repair against disk protection(CASSANDRA-20045) * BEGIN TRANSACTION crashes if a mutation touches multiple rows (CASSANDRA-20844) * Fix version range check in MessagingService.getVersionOrdinal (CASSANDRA-20842) * Allow custom constraints to be loaded via SPI (CASSANDRA-20824) diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index a7efe5735a..acb0429476 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -2683,10 +2683,14 @@ storage_compatibility_mode: NONE # the given value. Defaults to disabled. # reject_repair_compaction_threshold: 1024 -# At least 20% of disk must be unused to run incremental repair. It is useful to avoid disks filling up during -# incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily. +# At least 20% of disk must be unused to run repair. It is useful to avoid disks filling up during +# repair as anti-compaction during repair may contribute to additional space temporarily. # if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) # then set the ratio to 0.0 +# repair_disk_headroom_reject_ratio: 0.2; + +# This is the deprecated config which was used to safeguard incremental repairs. Use repair_disk_headroom_reject_ratio +# instead as it safeguards against all repairs. # incremental_repair_disk_headroom_reject_ratio: 0.2; # Configuration for Auto Repair Scheduler. diff --git a/conf/cassandra_latest.yaml b/conf/cassandra_latest.yaml index 69b4c647f7..c987a0ce42 100644 --- a/conf/cassandra_latest.yaml +++ b/conf/cassandra_latest.yaml @@ -2374,11 +2374,11 @@ storage_compatibility_mode: NONE # the given value. Defaults to disabled. # reject_repair_compaction_threshold: 1024 -# At least 20% of disk must be unused to run incremental repair. It is useful to avoid disks filling up during -# incremental repair as anti-compaction during incremental repair may contribute to additional space temporarily. +# At least 20% of disk must be unused to run repair. It is useful to avoid disks filling up during +# repair as anti-compaction during repair may contribute to additional space temporarily. # if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) # then set the ratio to 0.0 -# incremental_repair_disk_headroom_reject_ratio: 0.2; +# repair_disk_headroom_reject_ratio: 0.2; # Configuration for Auto Repair Scheduler. # diff --git a/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc b/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc index d5701895d7..bd9dad5aa7 100644 --- a/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc +++ b/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc @@ -318,9 +318,9 @@ When enabling auto_repair, it is advisable to configure the top level `reject_re configuration in cassandra.yaml as a backpressure mechanism to reject new repairs on instances that have many pending compactions. -==== Tune `incremental_repair_disk_headroom_reject_ratio` +==== Tune `repair_disk_headroom_reject_ratio` -By default, incremental repairs will be rejected if less than 20% of disk is available. If one wishes to be +By default, repairs will be rejected if less than 20% of disk is available. If one wishes to be conservative this top level configuration could be increased to a larger value to prevent filling your data directories. == Table configuration diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index a22b40fa27..e931b5a9d9 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -370,9 +370,10 @@ public class Config // The number of executors to use for building secondary indexes public volatile int concurrent_index_builders = 2; - // at least 20% of disk must be unused to run incremental repair + // at least 20% of disk must be unused to run repair // if you want to disable this feature (the recommendation is not to, but if you want to disable it for whatever reason) then set the ratio to 0.0 - public volatile double incremental_repair_disk_headroom_reject_ratio = 0.2; + @Replaces(oldName = "incremental_repair_disk_headroom_reject_ratio") + public volatile double repair_disk_headroom_reject_ratio = 0.2; /** * @deprecated retry support removed on CASSANDRA-10992 diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index d605d0cd98..5811cd3243 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -5955,18 +5955,18 @@ public class DatabaseDescriptor return conf.auto_repair; } - public static double getIncrementalRepairDiskHeadroomRejectRatio() + public static double getRepairDiskHeadroomRejectRatio() { - return conf.incremental_repair_disk_headroom_reject_ratio; + return conf.repair_disk_headroom_reject_ratio; } - public static void setIncrementalRepairDiskHeadroomRejectRatio(double value) + public static void setRepairDiskHeadroomRejectRatio(double value) { if (value < 0.0 || value > 1.0) { - throw new IllegalArgumentException("Value must be >= 0 and <= 1 for incremental_repair_disk_headroom_reject_ratio"); + throw new IllegalArgumentException("Value must be >= 0 and <= 1 for repair_disk_headroom_reject_ratio"); } - conf.incremental_repair_disk_headroom_reject_ratio = value; + conf.repair_disk_headroom_reject_ratio = value; } @VisibleForTesting diff --git a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java index ea9742609f..2f80bedd31 100644 --- a/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java +++ b/src/java/org/apache/cassandra/repair/RepairMessageVerbHandler.java @@ -123,7 +123,7 @@ public class RepairMessageVerbHandler implements IVerbHandler sendFailureResponse(message); return; } - if (!ActiveRepairService.verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind, prepareMessage.isIncremental)) + if (!ActiveRepairService.verifyDiskHeadroomThreshold(prepareMessage.parentRepairSession, prepareMessage.previewKind)) { // error is logged in verifyDiskHeadroomThreshold state.phase.fail("Not enough disk headroom to perform incremental repair"); diff --git a/src/java/org/apache/cassandra/service/ActiveRepairService.java b/src/java/org/apache/cassandra/service/ActiveRepairService.java index bee41a7fa3..c297ab1912 100644 --- a/src/java/org/apache/cassandra/service/ActiveRepairService.java +++ b/src/java/org/apache/cassandra/service/ActiveRepairService.java @@ -670,7 +670,7 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai public Future prepareForRepair(TimeUUID parentRepairSession, InetAddressAndPort coordinator, Set endpoints, RepairOption options, boolean isForcedRepair, List columnFamilyStores) { - if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind(), options.isIncremental())) + if (!verifyDiskHeadroomThreshold(parentRepairSession, options.getPreviewKind())) failRepair(parentRepairSession, "Rejecting incoming repair, disk usage above threshold"); // failRepair throws exception if (!verifyCompactionsPendingThreshold(parentRepairSession, options.getPreviewKind())) @@ -730,11 +730,8 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai return promise; } - public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind, boolean isIncremental) + public static boolean verifyDiskHeadroomThreshold(TimeUUID parentRepairSession, PreviewKind previewKind) { - if (!isIncremental) // disk headroom is required for anti-compaction which is only performed by incremental repair - return true; - double diskUsage = DiskUsageMonitor.instance.getDiskUsage(); double rejectRatio = ActiveRepairService.instance().getIncrementalRepairDiskHeadroomRejectRatio(); @@ -1110,12 +1107,12 @@ public class ActiveRepairService implements IEndpointStateChangeSubscriber, IFai public double getIncrementalRepairDiskHeadroomRejectRatio() { - return DatabaseDescriptor.getIncrementalRepairDiskHeadroomRejectRatio(); + return DatabaseDescriptor.getRepairDiskHeadroomRejectRatio(); } public void setIncrementalRepairDiskHeadroomRejectRatio(double value) { - DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(value); + DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(value); } /** diff --git a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java index ee4fa293a2..d1ecd33b1e 100644 --- a/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java +++ b/test/unit/org/apache/cassandra/service/ActiveRepairServiceTest.java @@ -530,9 +530,9 @@ public class ActiveRepairServiceTest } } - public void testVerifyDiskHeadroomThresholdFullRepair() + public void testVerifyDefaultDiskHeadroomThreshold() { - Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, false)); + Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE)); } @Test @@ -540,9 +540,9 @@ public class ActiveRepairServiceTest { DiskUsageMonitor.instance = diskUsageMonitor; when(diskUsageMonitor.getDiskUsage()).thenReturn(1.0); - DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(1.0); + DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(1.0); - Assert.assertFalse(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true)); + Assert.assertFalse(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE)); } @Test @@ -550,9 +550,9 @@ public class ActiveRepairServiceTest { DiskUsageMonitor.instance = diskUsageMonitor; when(diskUsageMonitor.getDiskUsage()).thenReturn(0.0); - DatabaseDescriptor.setIncrementalRepairDiskHeadroomRejectRatio(0.0); + DatabaseDescriptor.setRepairDiskHeadroomRejectRatio(0.0); - Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE, true)); + Assert.assertTrue(ActiveRepairService.verifyDiskHeadroomThreshold(TimeUUID.maxAtUnixMillis(0), PreviewKind.NONE)); } @Test(expected = RuntimeException.class)