[tor-commits] [ooni-probe/develop] Make the starting of tests more robust.

[isis at torproject.org]

commit 1247a80ec046e9c3b132ca6b23f602ca852e6b1a
Author: Arturo Filastò <art at fuffa.org>
Date:   Wed Feb 27 18:12:13 2013 +0100

    Make the starting of tests more robust.
---
 ooni/nettest.py       |   11 +++++++----
 ooni/oonicli.py       |   18 +++++++++++++-----
 ooni/utils/geodata.py |   39 +++++++++++++++++++++++++++++++++++++++
 3 files changed, 59 insertions(+), 9 deletions(-)

diff --git a/ooni/nettest.py b/ooni/nettest.py
index 7108cd1..0260ca2 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -39,7 +39,11 @@ class NetTestLoader(object):
                 config.privacy.includecountry or \
                 config.privacy.includecity):
             log.msg("We will include some geo data in the report")
-            client_geodata = geoip.IPToLocation(config.probe_ip.address)
+            try:
+                client_geodata = geodata.IPToLocation(config.probe_ip)
+            except e.GeoIPDataFilesNotFound:
+                log.err("Unable to find the geoip data files")
+                client_geodata = {'city': None, 'countrycode': None, 'asn': None}
 
         if config.privacy.includeip:
             client_geodata['ip'] = config.probe_ip.address
@@ -52,8 +56,7 @@ class NetTestLoader(object):
             client_geodata['asn'] = 'AS0'
         elif 'asn' in client_geodata:
             # XXX this regexp should probably go inside of geodata
-            client_geodata['asn'] = \
-                    re.search('AS\d+', client_geodata['asn']).group(0)
+            client_geodata['asn'] = client_geodata['asn']
             log.msg("Your AS number is: %s" % client_geodata['asn'])
         else:
             client_geodata['asn'] = None
@@ -307,7 +310,7 @@ class NetTest(object):
         self.state.taskDone()
 
         if len(self.report.reporters) == 0:
-            raise AllReportersFailed
+            raise e.AllReportersFailed
 
         return report_results
 
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 73d7709..93bf8bf 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -35,8 +35,8 @@ class Options(usage.Options):
     optParameters = [["reportfile", "o", None, "report file name"],
                      ["testdeck", "i", None,
                          "Specify as input a test deck: a yaml file containig the tests to run an their arguments"],
-                     ["collector", "c", None,
-                         "Address of the collector of test results. (example: http://127.0.0.1:8888)"],
+                     ["collector", "c", 'httpo://nkvphnp3p6agi5qq.onion',
+                         "Address of the collector of test results. default: httpo://nkvphnp3p6agi5qq.onion"],
                      ["logfile", "l", None, "log file name"],
                      ["pcapfile", "O", None, "pcap file name"],
                      ["parallelism", "p", "10", "input parallelism"],
@@ -130,15 +130,22 @@ def runWithDirector():
     director = Director()
     d = director.start()
 
+    def director_startup_failed(failure):
+        log.err("Failed to start the director")
+        log.exception(failure)
+        reactor.stop()
+
     # Wait until director has started up (including bootstrapping Tor) before adding tess
     def post_director_start(_):
         for net_test_loader in test_list:
-            yaml_reporter = YAMLReporter(net_test_loader.testDetails)
+            test_details = net_test_loader.testDetails
+
+            yaml_reporter = YAMLReporter(test_details)
             reporters = [yaml_reporter]
 
             if global_options['collector']:
                 try:
-                    oonib_reporter = OONIBReporter(net_test_loader.testDetails,
+                    oonib_reporter = OONIBReporter(test_details,
                             global_options['collector'])
                     reporters.append(oonib_reporter)
                 except InvalidOONIBCollectorAddress:
@@ -152,7 +159,7 @@ def runWithDirector():
                 with open('collector') as f:
                     reporter_url = random.choice(f.readlines())
                     reporter_url = reporter_url.split('#')[0].strip()
-                    oonib_reporter = OONIBReporter(net_test_loader.testDetails, reporter_url)
+                    oonib_reporter = OONIBReporter(test_details, reporter_url)
                     reporters.append(oonib_reporter)
 
             log.debug("adding callback for startNetTest")
@@ -160,4 +167,5 @@ def runWithDirector():
         d.addCallback(shutdown)
 
     d.addCallback(post_director_start)
+    d.addErrback(director_startup_failed)
     reactor.run()
diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py
new file mode 100644
index 0000000..56ce05c
--- /dev/null
+++ b/ooni/utils/geodata.py
@@ -0,0 +1,39 @@
+import re
+import os
+
+from twisted.web.client import Agent
+from twisted.internet import reactor, defer, protocol
+
+from ooni.utils import log, net
+from ooni import config
+from ooni.errors import GeoIPDataFilesNotFound
+
+try:
+    import pygeoip
+except ImportError:
+    log.err("Unable to import pygeoip. We will not be able to run geo IP related measurements")
+
+def IPToLocation(ipaddr):
+    city_file = os.path.join(config.advanced.geoip_data_dir, 'GeoLiteCity.dat')
+    country_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIP.dat')
+    asn_file = os.path.join(config.advanced.geoip_data_dir, 'GeoIPASNum.dat')
+
+    location = {'city': None, 'countrycode': None, 'asn': None}
+    try:
+        city_dat = pygeoip.GeoIP(city_file)
+        location['city'] = city_dat.record_by_addr(ipaddr)['city']
+
+        country_dat = pygeoip.GeoIP(country_file)
+        location['countrycode'] = country_dat.country_code_by_addr(ipaddr)
+
+        asn_dat = pygeoip.GeoIP(asn_file)
+        asn = asn_dat.org_by_addr(ipaddr)
+        location['asn'] = re.search('AS\d+', asn).group(0)
+
+    except IOError:
+        log.err("Could not find GeoIP data files. Go into data/ "
+                "and run make geoip")
+        raise GeoIPDataFilesNotFound
+
+    return location
+