[tor-commits] [doctor/master] Report unreachable fallback directories

[atagar at torproject.org]

commit e9047744d7f25e807737574b7e87140cb109f187
Author: Damian Johnson <atagar at torproject.org>
Date:   Sat May 28 16:43:59 2016 -0700

    Report unreachable fallback directories
    
    New notifications requested by teor that provides a summary if more than 25% of
    the fallback directories are unreachable or slow. Notices are sent to teor,
    nickm, me, and #tor-bots...
    
      https://trac.torproject.org/projects/tor/ticket/18177
---
 consensus_health_checker.py | 11 ++----
 fallback_directories.py     | 83 +++++++++++++++++++++++++++++++++++++++++++++
 util.py                     | 28 +++++++++++++++
 3 files changed, 114 insertions(+), 8 deletions(-)

diff --git a/consensus_health_checker.py b/consensus_health_checker.py
index c1aed79..a34e972 100755
--- a/consensus_health_checker.py
+++ b/consensus_health_checker.py
@@ -9,7 +9,6 @@ Performs a variety of checks against the present votes and consensus.
 import collections
 import datetime
 import os
-import socket
 import time
 import traceback
 
@@ -735,14 +734,10 @@ def is_orport_reachable(latest_consensus, consensuses, votes):
       continue  # authority isn't in the consensus
 
     for address, port, is_ipv6 in desc.or_addresses:
-      orport_socket = socket.socket(socket.AF_INET6 if is_ipv6 else socket.AF_INET, socket.SOCK_STREAM)
+      issue = util.check_reachability(address, port)
 
-      try:
-        orport_socket.connect((address, port))
-      except Exception as exc:
-        issues.append(Issue(Runlevel.WARNING, 'UNABLE_TO_REACH_ORPORT', authority = authority.nickname, address = address, port = port, error = exc, to = [authority]))
-      finally:
-        orport_socket.close()
+      if issue:
+        issues.append(Issue(Runlevel.WARNING, 'UNABLE_TO_REACH_ORPORT', authority = authority.nickname, address = address, port = port, error = issue, to = [authority]))
 
   return issues
 
diff --git a/fallback_directories.py b/fallback_directories.py
new file mode 100755
index 0000000..b9d6cc1
--- /dev/null
+++ b/fallback_directories.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# Copyright 2016, Damian Johnson and The Tor Project
+# See LICENSE for licensing information
+
+"""
+Report for how many of our fallback directories are unreachable.
+"""
+
+import time
+import traceback
+
+from stem.descriptor.remote import DescriptorDownloader, FallbackDirectory
+
+import util
+
+log = util.get_logger('fallback_directories')
+
+NOTIFICATION_THRESHOLD = 25  # send notice if this percentage of fallbacks are unusable
+TO_ADDRESSES = ['atagar at torproject.org', 'teor2345 at gmail.com', 'nickm at torproject.org']
+EMAIL_SUBJECT = 'Fallback Directory Summary'
+
+EMAIL_BODY = """\
+%i/%i (%i%%) fallback directories have become slow or unresponsive...
+
+"""
+
+downloader = DescriptorDownloader(timeout = 30)
+
+
+def main():
+  try:
+    fallback_directories = FallbackDirectory.from_remote().values()
+    log.info('Retrieved %i fallback directories' % len(fallback_directories))
+  except IOError as exc:
+    raise IOError("Unable to determine tor's fallback directories: %s" % exc)
+
+  issues = []
+
+  for relay in fallback_directories:
+    if not util.is_reachable(relay.address, relay.or_port):
+      log.info('%s ORPort unreachable' % relay.fingerprint)
+      issues.append('%s => ORPort is unreachable (%s:%i)' % (relay.fingerprint, relay.address, relay.or_port))
+      continue
+
+    if not util.is_reachable(relay.address, relay.dir_port):
+      log.info('%s DirPort unreachable' % relay.fingerprint)
+      issues.append('%s => DirPort is unreachable (%s:%i)' % (relay.fingerprint, relay.address, relay.dir_port))
+      continue
+
+    if relay.orport_v6 and not util.is_reachable(relay.orport_v6[0], relay.orport_v6[1]):
+      log.info('%s IPv6 ORPort unreachable' % relay.fingerprint)
+      issues.append('%s => IPv6 ORPort is unreachable (%s:%i)' % (relay.fingerprint, relay.orport_v6[0], relay.orport_v6[1]))
+      continue
+
+    start = time.time()
+    downloader.get_consensus(endpoints = [(relay.address, relay.dir_port)]).run()
+    download_time = time.time() - start
+    log.info('%s download time was %0.1f seconds' % (relay.fingerprint, download_time))
+
+    if download_time > 15:
+      issues.append('%s => Downloading the consensus took %0.1f seconds' % (relay.fingerprint, download_time))
+
+  issue_percent = 100.0 * len(issues) / len(fallback_directories)
+  log.info('%i ssues found (%i%%)' % (len(issues), issue_percent))
+
+  if issue_percent >= NOTIFICATION_THRESHOLD:
+    log.info('Sending notification')
+    body = EMAIL_BODY % (len(issues), len(fallback_directories), 100.0 * len(issues) / len(fallback_directories))
+    util.send(EMAIL_SUBJECT, body = body + '\n'.join(['  * %s' % issue for issue in issues]), to = TO_ADDRESSES)
+
+    # notification for #tor-bots
+
+    body = '\n'.join(['[fallback-directories] %s' % issue for issue in issues])
+    util.send('Announce or', body = body, to = ['tor-misc at commit.noreply.org'])
+
+
+if __name__ == '__main__':
+  try:
+    main()
+  except:
+    msg = "fallback_directories.py failed with:\n\n%s" % traceback.format_exc()
+    log.error(msg)
+    util.send("Script Error", body = msg, to = [util.ERROR_ADDRESS])
diff --git a/util.py b/util.py
index 483181e..9d4adcb 100644
--- a/util.py
+++ b/util.py
@@ -4,11 +4,13 @@ Module for issuing email notifications to me via gmail.
 
 import logging
 import os
+import socket
 import smtplib
 
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
+import stem.util.connection
 import stem.util.log
 
 FROM_ADDRESS = 'atagar at torproject.org'
@@ -55,6 +57,32 @@ def get_logger(name):
   return log
 
 
+def is_reachable(address, port):
+  return check_reachability(address, port) == None
+
+
+def check_reachability(address, port):
+  """
+  Simple check to see if we can establish a connection to the given endpoint.
+
+  :param str address: IPv4 or IPv6 address to check
+  :param int port: port to check
+
+  :returns: **None** if the endpoint is reachable and a **str** describing the issue otherwise
+  """
+
+  socket_type = socket.AF_INET6 if stem.util.connection.is_valid_ipv6_address(address) else socket.AF_INET
+  test_socket = socket.socket(socket_type, socket.SOCK_STREAM)
+
+  try:
+    test_socket.connect((address, port))
+    return None
+  except Exception as exc:
+    return str(exc)
+  finally:
+    test_socket.close()
+
+
 def log_stem_debugging(name):
   """
   Logs trace level stem output to the given log file.