[tor-commits] [ooni-probe/master] Progress on implementing web_connectivity test

[art at torproject.org]

commit 9ab068b324290df91d4296f2f3388071df3bbac4
Author: Arturo Filastò <arturo at filasto.net>
Date:   Tue Feb 2 16:07:04 2016 +0100

    Progress on implementing web_connectivity test
---
 ooni/nettests/blocking/web_connectivity.py | 62 ++++++++++++++++++++----------
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index bcc3a28..bc23f6d 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -13,7 +13,7 @@ from twisted.python import usage
 from ooni.utils import log
 
 from ooni.utils.net import StringProducer, BodyReceiver
-from ooni.templates import httpt, dnst, tcpt
+from ooni.templates import httpt, dnst
 from ooni.errors import failureToString
 
 class TCPConnectProtocol(Protocol):
@@ -21,6 +21,7 @@ class TCPConnectProtocol(Protocol):
         self.transport.loseConnection()
 
 class TCPConnectFactory(Factory):
+    noisy = False
     def buildProtocol(self, addr):
         return TCPConnectProtocol()
 
@@ -74,6 +75,9 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         self.report['accessible'] = None
         self.report['blocking'] = None
 
+        self.report['control_failure'] = None
+        self.report['experiment_failure'] = None
+
         self.report['tcp_connect'] = [
         ]
 
@@ -83,8 +87,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
         self.control = {
             'tcp_connect': {},
-            'dns_consistency': [],
-            'http_requests': {
+            'dns': {
+                'ips': []
+            },
+            'http_request': {
                 'body_length': None,
                 'headers': {}
             }
@@ -93,7 +99,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     def dns_discovery(self):
         return self.performALookup(self.localOptions['dns-discovery'])
 
-    def dns_consistency(self):
+    def experiment_dns_query(self):
         return self.performALookup(self.hostname)
 
     def tcp_connect(self, socket):
@@ -119,6 +125,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         @d.addErrback
         def eb(failure):
             result['status']['success'] = False
+            result['status']['failure'] = failureToString(failure)
             self.report['tcp_connect'].append(result)
         return d
 
@@ -144,11 +151,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
     def experiment_http_get_request(self):
         return self.doRequest(self.input)
 
-    def compare_control_experiment(self, experiment_http_response,
-                                   experiment_dns_answers):
-
-        blocking = None
-
+    def compare_body_lengths(self, experiment_http_response):
         control_body_length = self.control['http_request']['body_length']
         experiment_body_length = len(experiment_http_response.body)
 
@@ -165,20 +168,22 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
         self.report['body_proportion'] = rel
         if rel > float(self.factor):
             self.report['body_length_match'] = True
+            return None
         else:
-            blocking = 'http'
             self.report['body_length_match'] = False
+            return 'http'
 
+    def compare_dns_experiments(self, experiment_dns_answers):
         control_ips = set(self.control['dns']['ips'])
         experiment_ips = set(experiment_dns_answers)
 
         if len(control_ips.intersection(experiment_ips)) > 0:
             self.report['dns_consistency'] = 'consistent'
         else:
-            if blocking is not None:
-                blocking = 'dns'
             self.report['dns_consistency'] = 'inconsistent'
 
+    def compare_tcp_experiments(self):
+        blocking = False
         for idx, result in enumerate(self.report['tcp_connect']):
             socket = "%s:%s" % (result['ip'], result['port'])
             control_status = self.control['tcp_connect'][socket]
@@ -189,14 +194,13 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
                 blocking = 'tcp_ip'
             else:
                 self.report['tcp_connect'][idx]['status']['blocked'] = False
-
-        self.report['blocking'] = blocking
+        return blocking
 
     @defer.inlineCallbacks
     def test_web_connectivity(self):
         results = yield defer.DeferredList([
             self.dns_discovery(),
-            self.dns_consistency()
+            self.experiment_dns_query()
         ])
 
         self.report['client_resolver'] = None
@@ -207,11 +211,29 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
 
         sockets = map(lambda x: "%s:80" % x, results[1][1])
 
-        dl = [self.control_request(sockets)]
+        control_request = self.control_request(sockets)
+        @control_request.addErrback
+        def control_err(failure):
+            self.report['control_failure'] = failureToString(failure)
+
+        dl = [control_request]
         for socket in sockets:
             dl.append(self.tcp_connect(socket))
-        yield defer.DeferredList(dl)
+        results = yield defer.DeferredList(dl)
+
+        experiment_http = self.experiment_http_get_request()
+        @experiment_http.addErrback
+        def experiment_err(failure):
+            self.report['experiment_failure'] = failureToString(failure)
+
+        experiment_http_response = yield experiment_http
+
+        if self.report['control_failure'] is None and \
+                self.report['experiment_failure'] is None:
+            self.compare_body_lenghts(experiment_http_response)
+
+        if self.report['control_failure'] is None:
+            self.compare_dns_experiments(experiment_dns_answers)
 
-        experiment_http_response = yield self.experiment_http_get_request()
-        self.compare_control_experiment(experiment_http_response,
-                                        experiment_dns_answers)
+        if self.report['control_failure'] is None:
+            self.compare_tcp_experiments()