[tor-commits] [fallback-scripts/master] script: Optionally limit the number of relays downloaded from Onionoo

[teor at torproject.org]

commit eb15fe6ccb9de3a3535570fbb9c6f95e7dc2edb7
Author: teor <teor at torproject.org>
Date:   Thu Jun 27 22:34:51 2019 +1000

    script: Optionally limit the number of relays downloaded from Onionoo
    
    Tested with:
    TOR_FB_ONIONOO_LIMIT=200 ./updateFallbackDirs.py
    
    Part of 29100.
---
 updateFallbackDirs.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/updateFallbackDirs.py b/updateFallbackDirs.py
index 23991ad..fe94554 100755
--- a/updateFallbackDirs.py
+++ b/updateFallbackDirs.py
@@ -186,6 +186,13 @@ ONIONOO = getenv_conf('TOR_FB_ONIONOO',
                       'https://onionoo.torproject.org/', str)
 #ONIONOO = 'https://onionoo.thecthulhu.com/'
 
+# How many rows should we request from OnionOO?
+# We ask Onionoo to exclude the slowest and most recent relays.
+# None means "all relays".
+# Set env TOR_FB_ONIONOO_LIMIT="None" to request all relays.
+ONIONOO_LIMIT = getenv_conf('TOR_FB_ONIONOO_LIMIT',
+                            None, opt(int))
+
 # Don't bother going out to the Internet, just use the files available locally,
 # even if they're very old
 LOCAL_FILES_ONLY = getenv_conf('TOR_FB_LOCAL_FILES_ONLY',
@@ -508,10 +515,15 @@ def datestr_to_datetime(datestr):
 def onionoo_fetch(what, **kwargs):
   params = kwargs
   params['type'] = 'relay'
-  #params['limit'] = 10
+  if ONIONOO_LIMIT is not None:
+    params['limit'] = str(ONIONOO_LIMIT)
   params['first_seen_days'] = '%d-'%(ADDRESS_AND_PORT_STABLE_DAYS)
   params['last_seen_days'] = '-%d'%(MAX_DOWNTIME_DAYS)
   params['flag'] = 'V2Dir'
+  # Get the relays with the highest consensus weight first,
+  # then use first_seen to get a stable order.
+  # The order is important when we're limiting the number of relays returned.
+  params['order'] = '-consensus_weight,first_seen'
   url = ONIONOO + what + '?' + urllib.urlencode(params)
 
   # Unfortunately, the URL is too long for some OS filenames,