natas_hw
/
grpc
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #23911 from ericgribkoff/remove_flake_fix 

Account for empty EDS update in test_round_robin
This commit is contained in:
Eric Gribkoff 2020-08-20 16:43:14 -07:00 committed by GitHub
parent cb507e0ed4 8c97bc639c
commit e203dd44ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 21 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
tools/run_tests/run_xds_tests.py
View File

@ -575,17 +575,27 @@ def test_round_robin(gcp, backend_service, instance_group):
threshold = 1
wait_until_all_rpcs_go_to_given_backends(instance_names,
_WAIT_FOR_STATS_SEC)
stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC)
requests_received = [stats.rpcs_by_peer[x] for x in stats.rpcs_by_peer]
total_requests_received = sum(requests_received)
if total_requests_received != _NUM_TEST_RPCS:
raise Exception('Unexpected RPC failures', stats)
expected_requests = total_requests_received / len(instance_names)
for instance in instance_names:
if abs(stats.rpcs_by_peer[instance] - expected_requests) > threshold:
raise Exception(
'RPC peer distribution differs from expected by more than %d '
'for instance %s (%s)' % (threshold, instance, stats))
# TODO(ericgribkoff) Delayed config propagation from earlier tests
# may result in briefly receiving an empty EDS update, resulting in failed
# RPCs. Retry distribution validation if this occurs; long-term fix is
# creating new backend resources for each individual test case.
max_attempts = 10
for i in range(max_attempts):
stats = get_client_stats(_NUM_TEST_RPCS, _WAIT_FOR_STATS_SEC)
requests_received = [stats.rpcs_by_peer[x] for x in stats.rpcs_by_peer]
total_requests_received = sum(requests_received)
if total_requests_received != _NUM_TEST_RPCS:
logger.info('Unexpected RPC failures, retrying: %s', stats)
continue
expected_requests = total_requests_received / len(instance_names)
for instance in instance_names:
if abs(stats.rpcs_by_peer[instance] -
expected_requests) > threshold:
raise Exception(
'RPC peer distribution differs from expected by more than %d '
'for instance %s (%s)' % (threshold, instance, stats))
return
raise Exception('RPC failures persisted through %d retries' % max_attempts)
def test_secondary_locality_gets_no_requests_on_partial_primary_failure(