From 1099bbbe867ef07f98ada852d699a145919082fc Mon Sep 17 00:00:00 2001 From: michael stack Date: Sat, 31 Jan 2026 13:49:46 -0800 Subject: [PATCH] Increase chaos test timeout and reduce data size further Test ran for 4045s on Joshua, exceeding the 3600s BulkDump timeout. S3 chaos compounds on each SST file, causing exponential retry delays. Changes: - Increase bulkdump_job_timeout from 3600s to 5400s - Reduce nodeCount from 300 to 200 (fewer SST files) - Reduce transactionsPerSecond from 200 to 150 Co-Authored-By: Claude Opus 4.5 --- .../BackupS3BlobBulkLoadRestoreWithChaos.toml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml b/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml index 7afd8a27d3..8aac2f31c8 100644 --- a/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml +++ b/tests/slow/BackupS3BlobBulkLoadRestoreWithChaos.toml @@ -5,9 +5,9 @@ # Chaos must be minimal to allow the backup snapshot to complete. # # Chaos configuration (minimal rates for BulkDump compatibility): -# - 1% error rate (very occasional S3 failures) -# - 2% throttling rate (light throttling) -# - 1% delay rate (minimal random delays) +# - 0.5% error rate (very occasional S3 failures) +# - 1% throttling rate (light throttling) +# - 0.5% delay rate (minimal random delays) # - 0% corruption rate (no corruption - BulkDump can't handle this) # # NOTE: SST file handling is only applicable when simulation randomly chooses @@ -53,8 +53,9 @@ disable_audit_storage_final_replica_check_in_sim = true max_trace_lines = 5000000 # Longer timeouts to handle chaos-induced delays # BulkDump with S3 chaos can take much longer than normal (10x+ on Linux vs macOS) -bulkdump_job_timeout = 3600 -bulkload_job_timeout = 3600 +# With chaos compounding on many SST files, 3600s wasn't enough - increase to 5400s +bulkdump_job_timeout = 5400 +bulkload_job_timeout = 5400 [[flow_knobs]] MAX_BUGGIFIED_DELAY = 0.0 @@ -90,8 +91,9 @@ timeout = 7200 testName = 'Cycle' # Reduced to limit S3 operations during chaos # More nodes = more SST files = more chances for S3 failures to compound - nodeCount = 300 - transactionsPerSecond = 200.0 + # 200 nodes is minimum to still exercise meaningful BulkDump behavior + nodeCount = 200 + transactionsPerSecond = 150.0 testDuration = 30.0 expectedRate = 0