Make port collision detection more robust in fdb_cluster_fixture (#13249)
* Fix port retry in fdb_cluster_fixture by grepping stderr separately The blob_backup_restore_tests ctest fails on busy CI instances when ports are already in use by parallel tests. The existing loop retried on port-in-use errors by grepping the output file for 'Local address in use', but both stdout and stderr were tee'd to the same output file via async process substitution, causing races where the grep misses the error string and the loop exits prematurely. Fix: tee stderr to a separate file (output.stderr) so the grep for 'Local address in use' is reliable and not corrupted by interleaved stdout. Non-port-in-use failures still fail immediately with stderr printed for debugging. * Address feedback. Simplify stdout/stderr capture in fdb_cluster_fixture Replace process substitution with tee with plain file redirects. The tee was unnecessary since neither stream needs real-time terminal output: stdout is only grepped for PIDs and stderr is only checked for port collisions or dumped on failure.
This commit is contained in:
parent
07d35a32a9
commit
7d04d9c03c
|
|
@ -117,9 +117,7 @@ function start_fdb_cluster {
|
|||
local port_prefix=1500
|
||||
while : ; do
|
||||
port_prefix="$(( port_prefix + 100 ))"
|
||||
# Disable exit on error temporarily so can capture result from run cluster.
|
||||
# Then redirect the output of the run_customer_cluster.sh via tee via
|
||||
# 'process substitution'; piping to tee hangs on success.
|
||||
# Disable exit on error temporarily so we can capture the exit status.
|
||||
set +o errexit # a.k.a. set +e
|
||||
set +o noclobber
|
||||
# In the below $knobs will pick up single quotes -- its what bash does when it
|
||||
|
|
@ -135,8 +133,7 @@ function start_fdb_cluster {
|
|||
--stateless_count 1 --replication_count 1 --logs_count 1 \
|
||||
--storage_count "${ss_count}" --storage_type ssd-rocksdb-v1 \
|
||||
--dump_pids on \
|
||||
> >(tee "${output}") \
|
||||
2> >(tee "${output}" >&2)
|
||||
> "${output}" 2> "${output}.stderr"
|
||||
status="$?"
|
||||
# Restore exit on error.
|
||||
set -o errexit # a.k.a. set -e
|
||||
|
|
@ -171,13 +168,14 @@ function start_fdb_cluster {
|
|||
fi
|
||||
break;
|
||||
fi
|
||||
# Otherwise, look for 'Local address in use' and if found retry with different ports.
|
||||
# Use grep -a to treat binary files as text
|
||||
if grep -a 'Local address in use' "${output}"; then
|
||||
log "Ports in use; retry cluster start but with different ports"
|
||||
# Check stderr for port-in-use error. Only retry on port collision;
|
||||
# fail on anything else.
|
||||
if grep -a 'Local address in use' "${output}.stderr"; then
|
||||
log "Port ${port_prefix} in use, retrying with next port"
|
||||
continue
|
||||
fi
|
||||
err "Failed to start fdb cluster"
|
||||
err "Failed to start fdb cluster (stderr follows):"
|
||||
cat "${output}.stderr" >&2
|
||||
return 1
|
||||
done
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue