[tor-commits] [ooni-probe/master] Implement basic web connectivity test
art at torproject.org
art at torproject.org
Mon May 30 16:28:32 UTC 2016
commit d92a95b628167043a84ceabac77e45f3cd319d3d
Author: Arturo Filastò <arturo at filasto.net>
Date: Wed Feb 3 19:36:11 2016 +0100
Implement basic web connectivity test
* Various hacky fixes to enforce correct report format and handling of report_id
---
ooni/director.py | 5 ++
ooni/nettest.py | 6 ++-
ooni/nettests/blocking/web_connectivity.py | 86 ++++++++++++++++++++++++------
ooni/reporter.py | 7 ++-
ooni/tasks.py | 3 ++
ooni/templates/httpt.py | 19 +++----
6 files changed, 94 insertions(+), 32 deletions(-)
diff --git a/ooni/director.py b/ooni/director.py
index 02619c2..13a4503 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -258,6 +258,11 @@ class Director(object):
yield net_test.report.open()
+ # XXX this needs some serious refactoring
+ net_test_loader.reportID = report.reportID
+ net_test.reportID = report.reportID
+ net_test.testDetails['report_id'] = report.reportID
+
yield net_test.initializeInputProcessor()
try:
self.activeNetTests.append(net_test)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 074e8c2..166451a 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -161,6 +161,8 @@ class NetTestLoader(object):
method_prefix = 'test'
collector = None
yamloo = True
+ requiresTor = False
+ reportID = None
def __init__(self, options, test_file=None, test_string=None,
annotations={}):
@@ -568,10 +570,12 @@ class NetTest(object):
@defer.inlineCallbacks
def initializeInputProcessor(self):
- for test_class, test_method in self.testCases:
+ for test_class, _ in self.testCases:
test_class.inputs = yield defer.maybeDeferred(
test_class().getInputProcessor
)
+ if not test_class.inputs:
+ test_class.inputs = [None]
def generateMeasurements(self):
"""
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index bc23f6d..8d83ad9 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -3,6 +3,8 @@
import json
from urlparse import urlparse
+from ipaddr import IPv4Address, AddressValueError
+
from twisted.internet import reactor
from twisted.internet.protocol import Factory, Protocol
from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -10,6 +12,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet import defer
from twisted.python import usage
+from ooni import geoip
from ooni.utils import log
from ooni.utils.net import StringProducer, BodyReceiver
@@ -34,6 +37,16 @@ class UsageOptions(usage.Options):
]
+def is_public_ipv4_address(address):
+ try:
+ ip_address = IPv4Address(address)
+ if not any([ip_address.is_private,
+ ip_address.is_loopback]):
+ return True
+ return False
+ except AddressValueError:
+ return None
+
class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
"""
Web connectivity
@@ -51,8 +64,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
]
requiredTestHelpers = {
- 'backend': 'web_connectivity',
- 'dns-discovery': 'dns_discovery'
+ 'backend': 'web-connectivity',
+ 'dns-discovery': 'dns-discovery'
}
requiresRoot = False
requiresTor = False
@@ -78,8 +91,8 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
self.report['control_failure'] = None
self.report['experiment_failure'] = None
- self.report['tcp_connect'] = [
- ]
+ self.report['tcp_connect'] = []
+ self.report['control'] = {}
self.hostname = urlparse(self.input).netloc
if not self.hostname:
@@ -147,6 +160,7 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
response.deliverBody(BodyReceiver(finished, content_length))
body = yield finished
self.control = json.loads(body)
+ self.report['control'] = self.control
def experiment_http_get_request(self):
return self.doRequest(self.input)
@@ -168,33 +182,48 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
self.report['body_proportion'] = rel
if rel > float(self.factor):
self.report['body_length_match'] = True
- return None
+ return True
else:
self.report['body_length_match'] = False
- return 'http'
+ return False
def compare_dns_experiments(self, experiment_dns_answers):
control_ips = set(self.control['dns']['ips'])
experiment_ips = set(experiment_dns_answers)
+ for experiment_ip in experiment_ips:
+ if is_public_ipv4_address(experiment_ip) is False:
+ self.report['dns_consistency'] = 'inconsistent'
+ return False
+
if len(control_ips.intersection(experiment_ips)) > 0:
self.report['dns_consistency'] = 'consistent'
- else:
- self.report['dns_consistency'] = 'inconsistent'
+ return True
+
+ experiment_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
+ experiment_ips))
+ control_asns = set(map(lambda x: geoip.IPToLocation(x)['asn'],
+ control_ips))
+
+ if len(control_asns.intersection(experiment_asns)) > 0:
+ self.report['dns_consistency'] = 'consistent'
+ return True
+
+ self.report['dns_consistency'] = 'inconsistent'
+ return False
def compare_tcp_experiments(self):
- blocking = False
+ success = True
for idx, result in enumerate(self.report['tcp_connect']):
socket = "%s:%s" % (result['ip'], result['port'])
control_status = self.control['tcp_connect'][socket]
- log.debug(str(result))
if result['status']['success'] == False and \
control_status['status'] == True:
self.report['tcp_connect'][idx]['status']['blocked'] = True
- blocking = 'tcp_ip'
+ success = False
else:
self.report['tcp_connect'][idx]['status']['blocked'] = False
- return blocking
+ return success
@defer.inlineCallbacks
def test_web_connectivity(self):
@@ -208,8 +237,10 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
self.report['client_resolver'] = results[0][1][1]
experiment_dns_answers = results[1][1]
-
- sockets = map(lambda x: "%s:80" % x, results[1][1])
+ sockets = []
+ for answer in experiment_dns_answers:
+ if is_public_ipv4_address(answer) is True:
+ sockets.append("%s:80" % answer)
control_request = self.control_request(sockets)
@control_request.addErrback
@@ -228,12 +259,33 @@ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
experiment_http_response = yield experiment_http
+ blocking = None
+ body_length_match = None
+ dns_consistent = None
+ tcp_connect = None
+
if self.report['control_failure'] is None and \
self.report['experiment_failure'] is None:
- self.compare_body_lenghts(experiment_http_response)
+ body_length_match = self.compare_body_lengths(experiment_http_response)
if self.report['control_failure'] is None:
- self.compare_dns_experiments(experiment_dns_answers)
+ dns_consistent = self.compare_dns_experiments(experiment_dns_answers)
if self.report['control_failure'] is None:
- self.compare_tcp_experiments()
+ tcp_connect = self.compare_tcp_experiments()
+
+ if dns_consistent == True and tcp_connect == False:
+ blocking = 'tcp_ip'
+
+ elif dns_consistent == True and \
+ tcp_connect == True and body_length_match == False:
+ blocking = 'http'
+
+ elif dns_consistent == False:
+ blocking = 'dns'
+
+ self.report['blocking'] = blocking
+ if blocking is not None:
+ log.msg("Blocking detected on %s due to %s" % (self.input, blocking))
+ else:
+ log.msg("No blocking detected on %s" % self.input)
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 66a3e86..49df443 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -4,6 +4,8 @@ import json
import os
import re
+from copy import deepcopy
+
from datetime import datetime
from contextlib import contextmanager
@@ -192,9 +194,9 @@ class YAMLReporter(OReporter):
log.debug("Writing report with YAML reporter")
content = '---\n'
if isinstance(entry, Measurement):
- report_entry = entry.testInstance.report
+ report_entry = deepcopy(entry.testInstance.report)
elif isinstance(entry, dict):
- report_entry = entry
+ report_entry = deepcopy(entry)
else:
raise Exception("Failed to serialise entry")
content += safe_dump(report_entry)
@@ -576,6 +578,7 @@ class OONIBReportLog(object):
class Report(object):
+ reportID = None
def __init__(self, test_details, report_filename,
reportEntryManager, collector_address=None,
diff --git a/ooni/tasks.py b/ooni/tasks.py
index 72e211f..2c4f07d 100644
--- a/ooni/tasks.py
+++ b/ooni/tasks.py
@@ -115,6 +115,9 @@ class Measurement(TaskWithTimeout):
if 'input' not in self.testInstance.report.keys():
self.testInstance.report['input'] = test_input
+ if 'test_start_time' not in self.testInstance.report.keys():
+ start_time = otime.epochToNewTimestamp(self.testInstance._start_time)
+ self.testInstance.report['test_start_time'] = start_time
self.testInstance.setUp()
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 73bd5db..2b280f0 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -1,10 +1,9 @@
import re
import random
-from twisted.internet import defer
-
from txtorcon.interface import StreamListenerMixin
+from twisted.web.client import readBody, PartialDownloadError
from twisted.internet import reactor
from twisted.internet.endpoints import TCP4ClientEndpoint
from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent
@@ -13,7 +12,7 @@ from ooni.nettest import NetTestCase
from ooni.utils import log, base64Dict
from ooni.settings import config
-from ooni.utils.net import BodyReceiver, StringProducer, userAgents
+from ooni.utils.net import StringProducer, userAgents
from ooni.utils.trueheaders import TrueHeaders
from ooni.errors import handleAllFailures
@@ -197,6 +196,8 @@ class HTTPTest(NetTestCase):
return response
def _processResponseBodyFail(self, failure, request, response):
+ if failure.check(PartialDownloadError):
+ return failure.value.response
failure_string = handleAllFailures(failure)
HTTPTest.addToReport(self, request, response,
failure_string=failure_string)
@@ -281,17 +282,11 @@ class HTTPTest(NetTestCase):
else:
self.processResponseHeaders(response_headers_dict)
- try:
- content_length = int(response.headers.getRawHeaders('content-length')[0])
- except Exception:
- content_length = None
-
- finished = defer.Deferred()
- response.deliverBody(BodyReceiver(finished, content_length))
- finished.addCallback(self._processResponseBody, request,
- response, body_processor)
+ finished = readBody(response)
finished.addErrback(self._processResponseBodyFail, request,
response)
+ finished.addCallback(self._processResponseBody, request,
+ response, body_processor)
return finished
def doRequest(self, url, method="GET",
More information about the tor-commits
mailing list