jffwang
/
hetu-core
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!1410 issue# 307 fix: Failure detection by maximum retry fail doesnt take the exact value set in exchange.max-retry-count config parameter 

Merge pull request !1410 from i-robot/pull308
This commit is contained in:
i-robot 2022-03-15 09:08:32 +00:00 committed by Gitee
parent 54154ba4fe 5df46ebd06
commit 69b713cd7b
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
3 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hetu-docs/en/admin/properties.md
View File

@ -389,7 +389,7 @@ Exchanges transfer data between openLooKeng nodes for different stages of a quer
> - **Type:** `integer`
> - **Default value:** `10`
>
> The maximum number of retry for failed task performed by the coordinator before considering it as a permanent failure. This property is used only when exchange.is-timeout-failure-detection-enabled is set to false.
> The maximum number of retry for failed task performed by the coordinator before considering it as a permanent failure. This property is used only when exchange.is-timeout-failure-detection-enabled is set to false. This value needs to be atleast 3 (minimum retry count) to take effect.
### `sink.max-buffer-size`

29
presto-main/src/main/java/io/prestosql/server/remotetask/Backoff.java
View File

@ -75,14 +75,13 @@ public class Backoff
public Backoff(int minTries, Duration maxFailureInterval, Ticker ticker, List<Duration> backoffDelayIntervals, int maxTries)
{
checkArgument(minTries > 0, "minTries must be at least 1");
checkArgument(maxTries > 0, "maxTries must be at least 1");
requireNonNull(maxFailureInterval, "maxFailureInterval is null");
requireNonNull(ticker, "ticker is null");
requireNonNull(backoffDelayIntervals, "backoffDelayIntervals is null");
checkArgument(!backoffDelayIntervals.isEmpty(), "backoffDelayIntervals must contain at least one entry");
this.minTries = minTries;
this.maxTries = maxTries;
this.maxTries = (minTries < maxTries) ? maxTries : minTries;
this.maxFailureIntervalNanos = maxFailureInterval.roundTo(NANOSECONDS);
this.ticker = ticker;
this.backoffDelayIntervalsNanos = backoffDelayIntervals.stream()
@ -100,7 +99,7 @@ public class Backoff
checkArgument(!backoffDelayIntervals.isEmpty(), "backoffDelayIntervals must contain at least one entry");
this.minTries = minTries;
this.maxTries = minTries + MAX_RETRIES; // to guaranty higher max retry value when min retry is > MAX_RETRIES
this.maxTries = (minTries < MAX_RETRIES) ? MAX_RETRIES : minTries;
this.maxFailureIntervalNanos = maxFailureInterval.roundTo(NANOSECONDS);
this.ticker = ticker;
this.backoffDelayIntervalsNanos = backoffDelayIntervals.stream()
@ -136,10 +135,19 @@ public class Backoff
{
lastRequestStart = 0;
firstFailureTime = 0;
failureCount = 0;
setFailureCount(0, false);
lastFailureTime = 0;
}
private synchronized void setFailureCount(int n, boolean isInc)
{
if (isInc) {
failureCount = failureCount + n;
return;
}
failureCount = n;
}
/**
* @return true if max retry failed, now it is time to check node status from HeartbeatFailureDetector
*/
@ -148,7 +156,7 @@ public class Backoff
long now = ticker.read();
lastFailureTime = now;
failureCount++;
setFailureCount(1, true);
if (lastRequestStart != 0) {
failureRequestTimeTotal += now - lastRequestStart;
lastRequestStart = 0;
@ -160,11 +168,10 @@ public class Backoff
return false;
}
if (failureCount < minTries) {
if (getFailureCount() < minTries) {
return false;
}
return failureCount > maxTries;
return getFailureCount() >= maxTries;
}
/**
@ -186,7 +193,7 @@ public class Backoff
long now = ticker.read();
lastFailureTime = now;
failureCount++;
setFailureCount(1, true);
if (lastRequestStart != 0) {
failureRequestTimeTotal += now - lastRequestStart;
lastRequestStart = 0;
@ -198,7 +205,7 @@ public class Backoff
return false;
}
if (failureCount < minTries) {
if (getFailureCount() < minTries) {
return false;
}
@ -208,7 +215,7 @@ public class Backoff
public synchronized long getBackoffDelayNanos()
{
int tmpFailureCount = (int) min(backoffDelayIntervalsNanos.length, this.failureCount);
int tmpFailureCount = (int) min(backoffDelayIntervalsNanos.length, getFailureCount());
if (tmpFailureCount == 0) {
return 0;
}

4
presto-main/src/test/java/io/prestosql/operator/TestHttpPageBufferClient.java
View File

@ -459,9 +459,9 @@ public class TestHttpPageBufferClient
assertEquals(callback.getPages().size(), 0);
assertEquals(callback.getCompletedRequests(), 11);
assertEquals(callback.getFinishedBuffers(), 0);
assertEquals(callback.getFailedBuffers(), 1);
assertEquals(callback.getFailedBuffers(), 2);
assertInstanceOf(callback.getFailure(), PageTransportTimeoutException.class);
assertContains(callback.getFailure().getMessage(), WORKER_NODE_ERROR + " (http://localhost:8080/0 - 11 failures,");
assertContains(callback.getFailure().getMessage(), WORKER_NODE_ERROR + " (http://localhost:8080/0 - 10 failures,");
assertStatus(client, location, "queued", 0, 11, 11, 11, "not scheduled");
}