natas_hw
/
cassandra
mirror of https://github.com/apache/cassandra
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #69 from sampreethb/sampreethb/CASSANDRA-21115/auto_repair_v2_on_4_1 

CASSANDRA-21115: Fix initial auto-repairs skipped by too soon check (backport to auto_repair_v2_on_4_1)
This commit is contained in:
jaydeepkumar1984 2026-07-01 21:37:01 -07:00 committed by GitHub
parent 02b9c89786 84ce9f4f82
commit ebc10e9ade
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 189 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGES.txt
View File

@ -1,4 +1,5 @@
4.1.6
* Fix initial auto-repairs skipped by too soon check (CASSANDRA-21115)
* Improved observability in AutoRepair to report both expected vs. actual repair bytes and expected vs. actual keyspaces (CASSANDRA-20581)
* Stop repair scheduler if two major versions are detected (CASSANDRA-20048)
* AutoRepair: Safeguard Full repair against disk protection(CASSANDRA-20045)

30
src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java
View File

@ -210,7 +210,8 @@ public class AutoRepair
long startTimeInMillis = timeFunc.get();
logger.info("My host id: {}, my turn to run repair...repair primary-ranges only? {}", myId,
config.getRepairPrimaryTokenRangeOnly(repairType));
AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, timeFunc.get(), turn);
AutoRepairUtils.updateStartAutoRepairHistory(repairType, myId, startTimeInMillis, turn);
repairState.setLastRepairStartTime(startTimeInMillis);
repairState.setRepairKeyspaceCount(0);
repairState.setRepairInProgress(true);
@ -403,20 +404,31 @@ public class AutoRepair
}
}
private boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config, UUID myId)
@VisibleForTesting
boolean tooSoonToRunRepair(AutoRepairConfig.RepairType repairType, AutoRepairState repairState, AutoRepairConfig config, UUID myId)
{
if (repairState.getLastRepairTime() == 0)
if (repairState.getLastRepairFinishTime() == 0)
{
// the node has either just boooted or has not run repair before,
// the node has either just booted or has not run repair before,
// we should check for the node's repair history in the DB
repairState.setLastRepairTime(AutoRepairUtils.getLastRepairTimeForNode(repairType, myId));
repairState.setLastRepairFinishTime(AutoRepairUtils.getLastRepairFinishTimeForNode(repairType, myId));
repairState.setLastRepairStartTime(AutoRepairUtils.getLastRepairStartTimeForNode(repairType, myId));
}
// If repair has not completed (start >= finish), don't skip - allow it to continue/resume
if (repairState.getLastRepairStartTime() >= repairState.getLastRepairFinishTime())
{
logger.info("Incomplete or unstarted repair detected (start_ts={} >= finish_ts={}), allowing resume",
repairState.getLastRepairStartTime(), repairState.getLastRepairFinishTime());
return false;
}
/*
* check if it is too soon to run repair. one of the reason we
* should not run frequent repair is that repair triggers
* memtable flush
*/
long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairTime());
long timeElapsedSinceLastRepair = TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() - repairState.getLastRepairFinishTime());
if (timeElapsedSinceLastRepair < config.getRepairMinInterval(repairType).toSeconds())
{
logger.info("Too soon to run repair, last repair was done {} seconds ago",
@ -498,14 +510,14 @@ public class AutoRepair
"repairTokenRangesSkipCount {}, repairTablesSkipCount {}", repairType, timeInHours, repairState.getRepairKeyspaceCount(),
repairState.getSucceededTokenRangesCount(), repairState.getFailedTokenRangesCount(),
repairState.getSkippedTokenRangesCount(), repairState.getSkippedTablesCount());
if (repairState.getLastRepairTime() != 0)
if (repairState.getLastRepairFinishTime() != 0)
{
repairState.setClusterRepairTimeInSec((int) TimeUnit.MILLISECONDS.toSeconds(timeFunc.get() -
repairState.getLastRepairTime()));
repairState.getLastRepairFinishTime()));
logger.info("Cluster repair time for repair type {}: {} day(s)", repairType,
TimeUnit.SECONDS.toDays(repairState.getClusterRepairTimeInSec()));
}
repairState.setLastRepairTime(timeFunc.get());
repairState.setLastRepairFinishTime(timeFunc.get());
repairState.setRepairInProgress(false);
AutoRepairUtils.updateFinishAutoRepairHistory(repairType, myId, timeFunc.get());

22
src/java/org/apache/cassandra/repair/autorepair/AutoRepairState.java
View File

@ -64,7 +64,9 @@ public abstract class AutoRepairState
@VisibleForTesting
protected int totalTablesConsideredForRepair = 0;
@VisibleForTesting
protected long lastRepairTimeInMs;
protected long lastRepairFinishTimeInMs;
@VisibleForTesting
protected long lastRepairStartTimeInMs;
@VisibleForTesting
protected int nodeRepairTimeInSec = 0;
@VisibleForTesting
@ -120,9 +122,9 @@ public abstract class AutoRepairState
setTotalKeyspaceRepairPlansToRepair(repairPlans.stream().mapToInt(repairPlan -> repairPlan.getKeyspaceRepairPlans().size()).sum());
}
public long getLastRepairTime()
public long getLastRepairFinishTime()
{
return lastRepairTimeInMs;
return lastRepairFinishTimeInMs;
}
public void setTotalTablesConsideredForRepair(int count)
@ -135,9 +137,19 @@ public abstract class AutoRepairState
return totalTablesConsideredForRepair;
}
public void setLastRepairTime(long lastRepairTime)
public void setLastRepairFinishTime(long lastRepairFinishTime)
{
lastRepairTimeInMs = lastRepairTime;
lastRepairFinishTimeInMs = lastRepairFinishTime;
}
public long getLastRepairStartTime()
{
return lastRepairStartTimeInMs;
}
public void setLastRepairStartTime(long lastRepairStartTime)
{
lastRepairStartTimeInMs = lastRepairStartTime;
}
public int getClusterRepairTimeInSec()

31
src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java
View File

@ -177,11 +177,16 @@ public class AutoRepairUtils
"SELECT %s FROM %s.%s WHERE %s = ? AND %s = ?", COL_REPAIR_FINISH_TS, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID);
final static String SELECT_LAST_REPAIR_START_TIME_FOR_NODE = String.format(
"SELECT %s FROM %s.%s WHERE %s = ? AND %s = ?", COL_REPAIR_START_TS, SchemaConstants.DISTRIBUTED_KEYSPACE_NAME,
SystemDistributedKeyspace.AUTO_REPAIR_HISTORY, COL_REPAIR_TYPE, COL_HOST_ID);
static ModificationStatement delStatementRepairHistory;
static SelectStatement selectStatementRepairHistory;
static ModificationStatement delStatementPriorityStatus;
static SelectStatement selectStatementRepairPriority;
static SelectStatement selectLastRepairTimeForNode;
static SelectStatement selectLastRepairStartTimeForNode;
static ModificationStatement addPriorityHost;
static ModificationStatement insertNewRepairHistoryStatement;
static ModificationStatement recordStartRepairHistoryStatement;
@ -207,6 +212,8 @@ public class AutoRepairUtils
.forInternalCalls());
selectLastRepairTimeForNode = (SelectStatement) QueryProcessor.getStatement(SELECT_LAST_REPAIR_TIME_FOR_NODE, ClientState
.forInternalCalls());
selectLastRepairStartTimeForNode = (SelectStatement) QueryProcessor.getStatement(SELECT_LAST_REPAIR_START_TIME_FOR_NODE, ClientState
.forInternalCalls());
delStatementPriorityStatus = (ModificationStatement) QueryProcessor.getStatement(DEL_REPAIR_PRIORITY, ClientState
.forInternalCalls());
addPriorityHost = (ModificationStatement) QueryProcessor.getStatement(ADD_PRIORITY_HOST, ClientState
@ -382,7 +389,8 @@ public class AutoRepairUtils
// this function will be called when a node bootstrap finished
UUID hostId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort());
// insert the data first
insertNewRepairHistory(repairType, currentTimeMillis(), currentTimeMillis());
long timestamp = currentTimeMillis();
insertNewRepairHistory(repairType, timestamp, timestamp);
setForceRepair(repairType, hostId);
}
@ -407,7 +415,7 @@ public class AutoRepairUtils
logger.info("Set force repair repair type: {}, node: {}", repairType, hostId);
}
public static long getLastRepairTimeForNode(RepairType repairType, UUID hostId)
public static long getLastRepairFinishTimeForNode(RepairType repairType, UUID hostId)
{
ResultMessage.Rows rows = selectLastRepairTimeForNode.execute(QueryState.forInternalCalls(),
QueryOptions.forInternalCalls(internalQueryCL,
@ -423,6 +431,22 @@ public class AutoRepairUtils
return repairTime.one().getLong(COL_REPAIR_FINISH_TS);
}
public static long getLastRepairStartTimeForNode(RepairType repairType, UUID hostId)
{
ResultMessage.Rows rows = selectLastRepairStartTimeForNode.execute(QueryState.forInternalCalls(),
QueryOptions.forInternalCalls(internalQueryCL,
Lists.newArrayList(
ByteBufferUtil.bytes(repairType.toString()),
ByteBufferUtil.bytes(hostId))),
Dispatcher.RequestTime.forImmediateExecution());
UntypedResultSet repairTime = UntypedResultSet.create(rows.result);
if (repairTime.isEmpty())
{
return 0;
}
return repairTime.one().getLong(COL_REPAIR_START_TS);
}
@VisibleForTesting
public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType, List<AutoRepairHistory> autoRepairHistories, UUID myId)
{
@ -836,7 +860,8 @@ public class AutoRepairUtils
if (!autoRepairHistoryIds.contains(hostId))
{
logger.info("{} for repair type {} doesn't exist in the auto repair history table, insert a new record.", repairType, hostId);
insertNewRepairHistory(repairType, hostId, currentTimeMillis(), currentTimeMillis());
long timestamp = currentTimeMillis();
insertNewRepairHistory(repairType, hostId, timestamp, timestamp);
}
}

36
test/unit/org/apache/cassandra/repair/autorepair/AutoRepairParameterizedTest.java
View File

@ -284,7 +284,7 @@ public class AutoRepairParameterizedTest extends CQLTester
AutoRepair.instance.repair(repairType);
assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair());
assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue());
long lastRepairTime = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
//if repair was done then lastRepairTime should be non-zero
Assert.assertTrue(String.format("Expected lastRepairTime > 0, actual value lastRepairTime %d",
lastRepairTime), lastRepairTime > 0);
@ -299,7 +299,7 @@ public class AutoRepairParameterizedTest extends CQLTester
//in the first round let repair run
config.setRepairMinInterval(repairType, "0s");
AutoRepair.instance.repair(repairType);
long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
int consideredTables = AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair();
Assert.assertNotEquals(String.format("Expected total repaired tables > 0, actual value %s ", consideredTables),
0, consideredTables);
@ -307,7 +307,7 @@ public class AutoRepairParameterizedTest extends CQLTester
//if repair was done in last 24 hours then it should not trigger another repair
config.setRepairMinInterval(repairType, "24h");
AutoRepair.instance.repair(repairType);
long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
Assert.assertEquals(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " +
"lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2);
assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair());
@ -322,14 +322,14 @@ public class AutoRepairParameterizedTest extends CQLTester
long prevCount = state.getTotalMVTablesConsideredForRepair();
AutoRepairService.instance.getAutoRepairConfig().setRepairMinInterval(repairType, "0s");
AutoRepair.instance.repair(repairType);
long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
Assert.assertTrue(String.format("Expected lastRepairTime1 > 0, actual value lastRepairTime1 %d",
lastRepairTime1), lastRepairTime1 > 0);
UUID myId = Gossiper.instance.getHostId(FBUtilities.getBroadcastAddressAndPort());
Assert.assertNotEquals("Expected my turn for the repair",
NOT_MY_TURN, AutoRepairUtils.myTurnToRunRepair(repairType, myId));
AutoRepair.instance.repair(repairType);
long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
Assert.assertNotSame(String.format("Expected repair time to be same, actual value lastRepairTime1 %d, " +
"lastRepairTime2 %d", lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2);
assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair());
@ -365,9 +365,9 @@ public class AutoRepairParameterizedTest extends CQLTester
long prevCount = state.getTotalMVTablesConsideredForRepair();
config.setRepairMinInterval(repairType, "0s");
config.setAutoRepairEnabled(repairType, false);
long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime1 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
AutoRepair.instance.repair(repairType);
long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime2 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
//Since repair has not happened, both the last repair times should be same
Assert.assertEquals(String.format("Expected lastRepairTime1 %d, and lastRepairTime2 %d to be same",
lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime2);
@ -375,7 +375,7 @@ public class AutoRepairParameterizedTest extends CQLTester
config.setAutoRepairEnabled(repairType, true);
AutoRepair.instance.repair(repairType);
//since repair is done now, so lastRepairTime1/lastRepairTime2 and lastRepairTime3 should not be same
long lastRepairTime3 = AutoRepair.instance.repairStates.get(repairType).getLastRepairTime();
long lastRepairTime3 = AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime();
Assert.assertNotSame(String.format("Expected lastRepairTime1 %d, and lastRepairTime3 %d to be not same",
lastRepairTime1, lastRepairTime2), lastRepairTime1, lastRepairTime3);
assertEquals(prevCount, state.getTotalMVTablesConsideredForRepair());
@ -411,19 +411,19 @@ public class AutoRepairParameterizedTest extends CQLTester
AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
config.setMaterializedViewRepairEnabled(repairType, true);
config.setRepairMinInterval(repairType, "0s");
AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis());
AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(System.currentTimeMillis());
AutoRepair.instance.repair(repairType);
assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair());
assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue());
config.setMaterializedViewRepairEnabled(repairType, false);
AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis());
AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(System.currentTimeMillis());
AutoRepair.instance.repair(repairType);
assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair());
assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue());
config.setMaterializedViewRepairEnabled(repairType, true);
AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(System.currentTimeMillis());
AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(System.currentTimeMillis());
AutoRepair.instance.repair(repairType);
assertEquals(1, AutoRepair.instance.repairStates.get(repairType).getTotalMVTablesConsideredForRepair());
assertEquals(1, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue());
@ -462,7 +462,7 @@ public class AutoRepairParameterizedTest extends CQLTester
config.setRepairSSTableCountHigherThreshold(repairType, 9);
assertEquals(0, state.getSkippedTokenRangesCount());
assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).skippedTokenRangesCount.getValue().intValue());
state.setLastRepairTime(0);
state.setLastRepairFinishTime(0);
AutoRepair.instance.repair(repairType);
assertEquals(0, state.getTotalMVTablesConsideredForRepair());
assertEquals(0, AutoRepairMetricsManager.getMetrics(repairType).totalMVTablesConsideredForRepair.getValue().intValue());
@ -474,7 +474,7 @@ public class AutoRepairParameterizedTest extends CQLTester
// set it to higher value, and this time, the tables should not be skipped
config.setRepairSSTableCountHigherThreshold(repairType, beforeCount);
state.setLastRepairTime(0);
state.setLastRepairFinishTime(0);
state.setSkippedTablesCount(0);
state.setTotalMVTablesConsideredForRepair(0);
AutoRepair.instance.repair(repairType);
@ -509,7 +509,7 @@ public class AutoRepairParameterizedTest extends CQLTester
timeFuncCalls++;
return timeFuncCalls * 1000L;
};
AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(1000L);
AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(1000L);
AutoRepair.instance.repair(repairType);
@ -544,7 +544,7 @@ public class AutoRepairParameterizedTest extends CQLTester
{
AutoRepairConfig config = AutoRepairService.instance.getAutoRepairConfig();
AutoRepair.instance.repairStates.put(repairType, autoRepairState);
when(autoRepairState.getLastRepairTime()).thenReturn((long) 0);
when(autoRepairState.getLastRepairFinishTime()).thenReturn((long) 0);
AtomicInteger getRepairRunnableCalls = new AtomicInteger();
AtomicReference<AutoRepair.RepairProgressListener> prevListener = new AtomicReference<>();
doAnswer(invocation -> {
@ -643,7 +643,7 @@ public class AutoRepairParameterizedTest extends CQLTester
config.setMaterializedViewRepairEnabled(repairType, true);
long lastRepairTime = System.currentTimeMillis() - 1000;
AutoRepairUtils.insertNewRepairHistory(repairType, 0, lastRepairTime);
AutoRepair.instance.repairStates.get(repairType).setLastRepairTime(0);
AutoRepair.instance.repairStates.get(repairType).setLastRepairFinishTime(0);
config.setRepairMinInterval(repairType, "1h");
AutoRepair.instance.repair(repairType);
@ -651,7 +651,7 @@ public class AutoRepairParameterizedTest extends CQLTester
// repair scheduler should not attempt to run repair as last repair time in DB is current time - 1s
assertEquals(0, AutoRepair.instance.repairStates.get(repairType).getTotalTablesConsideredForRepair());
// repair scheduler should load the repair time from the DB
assertEquals(lastRepairTime, AutoRepair.instance.repairStates.get(repairType).getLastRepairTime());
assertEquals(lastRepairTime, AutoRepair.instance.repairStates.get(repairType).getLastRepairFinishTime());
}
@Test
@ -879,7 +879,7 @@ public class AutoRepairParameterizedTest extends CQLTester
}
return runnable;
}).when(spyState).getRepairRunnable(any(), any(), any(), anyBoolean());
when(spyState.getLastRepairTime()).thenReturn((long) 0);
when(spyState.getLastRepairFinishTime()).thenReturn((long) 0);
AutoRepairService.instance.getAutoRepairConfig().setRepairMaxRetries(repairType, 0);
AutoRepair.instance.repairStates.put(repairType, spyState);

8
test/unit/org/apache/cassandra/repair/autorepair/AutoRepairStateTest.java
View File

@ -81,9 +81,9 @@ public class AutoRepairStateTest extends CQLTester
public void testGetLastRepairTime()
{
AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig());
state.lastRepairTimeInMs = 1;
state.lastRepairFinishTimeInMs = 1;
assertEquals(1, state.getLastRepairTime());
assertEquals(1, state.getLastRepairFinishTime());
}
@Test
@ -110,9 +110,9 @@ public class AutoRepairStateTest extends CQLTester
{
AutoRepairState state = RepairType.getAutoRepairState(repairType, new AutoRepairConfig());
state.setLastRepairTime(1);
state.setLastRepairFinishTime(1);
assertEquals(1, state.lastRepairTimeInMs);
assertEquals(1, state.lastRepairFinishTimeInMs);
}
@Test

78
test/unit/org/apache/cassandra/repair/autorepair/AutoRepairTest.java
View File

@ -20,6 +20,7 @@ package org.apache.cassandra.repair.autorepair;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import org.junit.Before;
import org.junit.BeforeClass;
@ -29,16 +30,24 @@ import org.junit.Assert;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.config.DurationSpec;
import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.QueryProcessor;
import org.apache.cassandra.db.Keyspace;
import org.apache.cassandra.exceptions.ConfigurationException;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.KeyspaceParams;
import org.apache.cassandra.schema.ReplicationParams;
import org.apache.cassandra.repair.autorepair.AutoRepairConfig.RepairType;
import org.apache.cassandra.schema.SchemaConstants;
import org.apache.cassandra.schema.SchemaTestUtil;
import org.apache.cassandra.schema.SystemDistributedKeyspace;
import org.apache.cassandra.service.AutoRepairService;
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.FBUtilities;
import static org.apache.cassandra.Util.setAutoRepairEnabled;
import static org.apache.cassandra.config.CassandraRelevantProperties.SYSTEM_DISTRIBUTED_DEFAULT_RF;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
/**
@ -49,8 +58,11 @@ public class AutoRepairTest extends CQLTester
@BeforeClass
public static void setupClass() throws Exception
{
SYSTEM_DISTRIBUTED_DEFAULT_RF.setInt(1);
setAutoRepairEnabled(true);
requireNetwork();
AutoRepairUtils.setup();
StorageService.instance.doAutoRepairSetup();
}
@Before
@ -159,4 +171,70 @@ public class AutoRepairTest extends CQLTester
}
}
}
@Test
public void testTooSoonToRunRepairAllowsResumeOfInProgressRepair()
{
RepairType repairType = RepairType.FULL;
UUID myId = StorageService.instance.getHostIdForEndpoint(FBUtilities.getBroadcastAddressAndPort());
long now = System.currentTimeMillis();
// Truncate history table to start fresh
QueryProcessor.executeInternal(String.format(
"TRUNCATE %s.%s",
SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY));
// Insert with start_ts > finish_ts to simulate in-progress repair (crashed before completion)
AutoRepairUtils.insertNewRepairHistory(repairType, myId, now, now - 1000);
AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig();
AutoRepairState repairState = RepairType.getAutoRepairState(repairType, config);
// Even though finish_ts was very recent, should return false because repair is in progress
assertFalse(AutoRepair.instance.tooSoonToRunRepair(repairType, repairState, config, myId));
}
@Test
public void testTooSoonToRunRepairReturnsTrueWhenRepairCompletedRecently()
{
RepairType repairType = RepairType.FULL;
UUID myId = StorageService.instance.getHostIdForEndpoint(FBUtilities.getBroadcastAddressAndPort());
long now = System.currentTimeMillis();
// Truncate history table to start fresh
QueryProcessor.executeInternal(String.format(
"TRUNCATE %s.%s",
SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY));
// Insert with finish_ts > start_ts to simulate completed repair
AutoRepairUtils.insertNewRepairHistory(repairType, myId, now - 1000, now);
AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig();
AutoRepairState repairState = RepairType.getAutoRepairState(repairType, config);
// Should return true because repair completed recently and min_repair_interval hasn't passed
assertTrue(AutoRepair.instance.tooSoonToRunRepair(repairType, repairState, config, myId));
}
@Test
public void testTooSoonToRunRepairAllowsResumeWhenStartEqualsFinish()
{
RepairType repairType = RepairType.FULL;
UUID myId = StorageService.instance.getHostIdForEndpoint(FBUtilities.getBroadcastAddressAndPort());
long now = System.currentTimeMillis();
// Truncate history table to start fresh
QueryProcessor.executeInternal(String.format(
"TRUNCATE %s.%s",
SchemaConstants.DISTRIBUTED_KEYSPACE_NAME, SystemDistributedKeyspace.AUTO_REPAIR_HISTORY));
// Insert with start_ts == finish_ts (edge case: repair never started progressing)
AutoRepairUtils.insertNewRepairHistory(repairType, myId, now, now);
AutoRepairConfig config = DatabaseDescriptor.getAutoRepairConfig();
AutoRepairState repairState = RepairType.getAutoRepairState(repairType, config);
// Should return false because repair is incomplete (start >= finish edge case)
assertFalse(AutoRepair.instance.tooSoonToRunRepair(repairType, repairState, config, myId));
}
}

24
test/unit/org/apache/cassandra/repair/autorepair/AutoRepairUtilsTest.java
View File

@ -576,7 +576,7 @@ public class AutoRepairUtilsTest extends CQLTester
AutoRepairUtils.insertNewRepairHistory(repairType, myID, currentMillis, currentMillis - 100);
AutoRepairUtils.insertNewRepairHistory(repairType, otherID, currentMillis, currentMillis + 100);
assertEquals(currentMillis - 100, AutoRepairUtils.getLastRepairTimeForNode(repairType, myID));
assertEquals(currentMillis - 100, AutoRepairUtils.getLastRepairFinishTimeForNode(repairType, myID));
}
@Test
@ -584,7 +584,27 @@ public class AutoRepairUtilsTest extends CQLTester
{
UUID myID = UUID.randomUUID();
assertEquals(0, AutoRepairUtils.getLastRepairTimeForNode(repairType, myID));
assertEquals(0, AutoRepairUtils.getLastRepairFinishTimeForNode(repairType, myID));
}
@Test
public void testGetLastRepairStartTimeForNode()
{
UUID myID = UUID.randomUUID();
UUID otherID = UUID.randomUUID();
long currentMillis = System.currentTimeMillis();
AutoRepairUtils.insertNewRepairHistory(repairType, myID, currentMillis, currentMillis - 100);
AutoRepairUtils.insertNewRepairHistory(repairType, otherID, currentMillis + 50, currentMillis + 100);
assertEquals(currentMillis, AutoRepairUtils.getLastRepairStartTimeForNode(repairType, myID));
}
@Test
public void testGetLastRepairStartTimeForNodeWhenHistoryIsEmpty()
{
UUID myID = UUID.randomUUID();
assertEquals(0, AutoRepairUtils.getLastRepairStartTimeForNode(repairType, myID));
}
@Test