[tor-commits] [chutney/master] verify: Wait for a consistent amount of time

[teor at torproject.org]

commit 95ce144c2615dd9400a41d8374e56683ad9975b6
Author: teor <teor at torproject.org>
Date:   Thu Mar 12 13:38:38 2020 +1000

    verify: Wait for a consistent amount of time
    
    Wait for a consistent amount of time between verify attempts,
    regardless of the time elapsed during the attempt.
    
    Try to make at least 2 verify attempts per consensus interval,
    but also try to limit the number of attempts.
    
    Also increase the default timeout: sometimes 3 seconds is too
    short on a heavily-loaded machine.
    
    These changes should make verification more reliable.
    
    Part of 33379.
---
 lib/chutney/Traffic.py          |  2 +-
 scripts/chutney_tests/verify.py | 31 ++++++++++++++++++++++---------
 torrc_templates/authority.i     |  2 ++
 3 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/lib/chutney/Traffic.py b/lib/chutney/Traffic.py
index 9990dcd..d8918ee 100755
--- a/lib/chutney/Traffic.py
+++ b/lib/chutney/Traffic.py
@@ -440,7 +440,7 @@ class TrafficTester(object):
         while now < end and not self.tests.all_done():
             # run only one iteration at a time, with a nice short timeout, so we
             # can actually detect completion and timeouts.
-            asyncore.loop(0.2, False, self.socket_map, 1)
+            asyncore.loop(5.0, False, self.socket_map, 1)
             now = time.time()
             if now > dump_at:
                 debug("Test status: %s"%self.tests.status())
diff --git a/scripts/chutney_tests/verify.py b/scripts/chutney_tests/verify.py
index a0f0c3e..2f048e0 100644
--- a/scripts/chutney_tests/verify.py
+++ b/scripts/chutney_tests/verify.py
@@ -7,6 +7,9 @@ import time
 import chutney
 
 def run_test(network):
+    # Try to verify twice each consensus
+    VERIFY_ATTEMPT_INTERVAL = network.V3_AUTH_VOTING_INTERVAL/2.0 - 1.0
+    TIMEOUT_INTERVAL = max(VERIFY_ATTEMPT_INTERVAL - 1.0, 5.0)
     wait_time = network._dfltEnv['bootstrap_time']
     start_time = time.time()
     end_time = start_time + wait_time
@@ -14,20 +17,28 @@ def run_test(network):
           % wait_time)
     status = False
     # Keep on retrying the verify until it succeeds or times out
-    while not status and time.time() < end_time:
+    now = start_time
+    while not status and now < end_time:
         # TrafficTester connections time out after ~3 seconds
-        # a TrafficTester times out after ~10 seconds if no data is being sent
-        status = _verify_traffic(network)
+        # a TrafficTester times out after ~6 seconds if no data is being sent
+        last_attempt_time = now
+        status = _verify_traffic(network, timeout=TIMEOUT_INTERVAL)
+        now = time.time()
+        elapsed_attempt_time = now - last_attempt_time
         # Avoid madly spewing output if we fail immediately each time
         if not status:
-            time.sleep(5)
+            # We want at least 2 verify attempts per consensus interval
+            sleep_time = VERIFY_ATTEMPT_INTERVAL - elapsed_attempt_time
+            if sleep_time > 0:
+                time.sleep(sleep_time)
+                now = time.time()
     print("Transmission: %s" % ("Success" if status else "Failure"))
     if not status:
         print("Set CHUTNEY_DEBUG to diagnose.")
     return status
 
 
-def _verify_traffic(network):
+def _verify_traffic(network, timeout=5.0):
     """Verify (parts of) the network by sending traffic through it
     and verify what is received."""
     # TODO: IPv6 SOCKSPorts, SOCKSPorts with IPv6Traffic, and IPv6 Exits
@@ -43,8 +54,7 @@ def _verify_traffic(network):
     # and a source-sink pair for a (bridge) client to each hidden service
     DATALEN = network._dfltEnv['data_bytes']
     # Print a dot each time a sink verifies this much data
-    DOTDATALEN = 5 * 1024 * 1024  # Octets.
-    TIMEOUT = 3                   # Seconds.
+    DOTDATALEN = 5 * 1024 * 1024  # Octets
     # Calculate the amount of random data we should use
     randomlen = _calculate_randomlen(DATALEN)
     reps = _calculate_reps(DATALEN, randomlen)
@@ -65,8 +75,11 @@ def _verify_traffic(network):
         tmpdata = {}
     # now make the connections
     bind_to = (LISTEN_ADDR, LISTEN_PORT)
-    tt = chutney.Traffic.TrafficTester(bind_to, tmpdata, TIMEOUT, reps,
-                                       dot_reps)
+    tt = chutney.Traffic.TrafficTester(bind_to,
+                                       data=tmpdata,
+                                       timeout=timeout,
+                                       repetitions=reps,
+                                       dot_repetitions=dot_reps)
     # _env does not implement get() due to its fallback to parent behaviour
     client_list = filter(lambda n:
                          n._env['tag'].startswith('c') or
diff --git a/torrc_templates/authority.i b/torrc_templates/authority.i
index e307bf5..e44b0b6 100644
--- a/torrc_templates/authority.i
+++ b/torrc_templates/authority.i
@@ -21,6 +21,8 @@ TestingV3AuthInitialVoteDelay 4
 TestingV3AuthInitialDistDelay 4
 # Vote + Dist must be less than Interval/2, because when there's no consensus,
 # tor uses Interval/2 as the voting interval
+#
+# Keep in sync with scripts/chutney_tests/verify.py V3_AUTH_VOTING_INTERVAL
 V3AuthVotingInterval 20
 V3AuthVoteDelay 4
 V3AuthDistDelay 4