Fix flaky test_short_disconnection_doesnt_stop_backup and test_short_disconnection_doesnt_stop_restore

alexey-milovidov · claude · alexey-milovidov · commit c79630ba890b · 2026-02-13T20:37:28.000+01:00
When `faster_zk_disconnect_detect.xml` is randomly chosen (which sets `session_timeout_ms=5000`), the ZK connection drop via iptables must be short enough to avoid session expiry. Previously, the drop duration was up to 3-4 seconds via `random_sleep`. Combined with the time since the last heartbeat (~1.7s for a 5s session timeout) and reconnection overhead, the total silence could exceed 5 seconds, causing the ZK session to expire and the backup/restore to fail regardless of the 30-second `failure_after_host_disconnected_for_seconds` threshold. The fix limits the drop duration to 1 second when using the faster ZK disconnect detection config, while keeping the original duration when using default ZK settings. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=96758&sha=d29b41fbe684f8c90ace4fd71828ce0d4ac8b88f&name_0=PR&name_1=Integration%20tests%20%28arm_binary%2C%20distributed%20plan%2C%204%2F4%29 Closes: #80359 Co-Authored-By: Claude Sonnet 4.5 (1M context) <noreply@anthropic.com>
diff --git a/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py b/tests/integration/test_backup_restore_on_cluster/test_cancel_backup.py
@@ -679,14 +679,17 @@ def test_short_disconnection_doesnt_stop_backup():
         assert get_num_system_processes(initiator, backup_id=backup_id) >= 1
 
         # Dropping connection for less than `failure_after_host_disconnected_for_seconds`
+        # When using faster_zk_disconnect_detect.xml (session_timeout_ms=5000),
+        # the drop duration must be short enough to avoid ZK session expiry.
+        max_drop_seconds = 1 if use_faster_zk_disconnect_detect else 4
         with PartitionManager() as pm:
             random_sleep(4)
             node_to_drop_zk_connection = random_node()
             print(
                 f"Dropping connection between {get_node_name(node_to_drop_zk_connection)} and ZooKeeper at {format_current_time()}"
             )
             pm.drop_instance_zk_connections(node_to_drop_zk_connection)
-            random_sleep(4)
+            random_sleep(max_drop_seconds)
             print(
                 f"Restoring connection between {get_node_name(node_to_drop_zk_connection)} and ZooKeeper at {format_current_time()}"
             )
@@ -733,14 +736,17 @@ def test_short_disconnection_doesnt_stop_restore():
         assert get_num_system_processes(initiator, restore_id=restore_id) >= 1
 
         # Dropping connection for less than `failure_after_host_disconnected_for_seconds`
+        # When using faster_zk_disconnect_detect.xml (session_timeout_ms=5000),
+        # the drop duration must be short enough to avoid ZK session expiry.
+        max_drop_seconds = 1 if use_faster_zk_disconnect_detect else 3
         with PartitionManager() as pm:
             random_sleep(3)
             node_to_drop_zk_connection = random_node()
             print(
                 f"Dropping connection between {get_node_name(node_to_drop_zk_connection)} and ZooKeeper at {format_current_time()}"
             )
             pm.drop_instance_zk_connections(node_to_drop_zk_connection)
-            random_sleep(3)
+            random_sleep(max_drop_seconds)
             print(
                 f"Restoring connection between {get_node_name(node_to_drop_zk_connection)} and ZooKeeper at {format_current_time()}"
             )
