[tor-commits] [ooni-probe/master] Implement a basic test that tests for reachability of the whatsapp servers
art at torproject.org
art at torproject.org
Fri Jan 13 12:39:57 UTC 2017
commit 3e9c0337b4142a884797784cfa9a94be423a2a09
Author: Arturo Filastò <arturo at filasto.net>
Date: Sun Jul 31 14:55:16 2016 +0200
Implement a basic test that tests for reachability of the whatsapp servers
---
ooni/nettest.py | 4 +
ooni/nettests/blocking/web_connectivity.py | 2 +-
ooni/nettests/blocking/whatsapp.py | 401 +++++++++++++++++++++++++++++
3 files changed, 406 insertions(+), 1 deletion(-)
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 150858c..5ff5485 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -268,6 +268,10 @@ class NetTestLoader(object):
parameter.pop()
self.usageOptions.optParameters.append(parameter)
+ if getattr(test_class.usageOptions, 'optFlags', None):
+ for parameter in test_class.usageOptions.optFlags:
+ self.usageOptions.optFlags.append(parameter)
+
if getattr(test_class, 'inputFile', None):
self.usageOptions.optParameters.append(test_class.inputFile)
diff --git a/ooni/nettests/blocking/web_connectivity.py b/ooni/nettests/blocking/web_connectivity.py
index 629ef1c..1d5cf1f 100644
--- a/ooni/nettests/blocking/web_connectivity.py
+++ b/ooni/nettests/blocking/web_connectivity.py
@@ -38,7 +38,7 @@ class UsageOptions(usage.Options):
]
-class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
+ class WebConnectivityTest(httpt.HTTPTest, dnst.DNSTest):
"""
Web connectivity
"""
diff --git a/ooni/nettests/blocking/whatsapp.py b/ooni/nettests/blocking/whatsapp.py
new file mode 100644
index 0000000..ac4ed1c
--- /dev/null
+++ b/ooni/nettests/blocking/whatsapp.py
@@ -0,0 +1,401 @@
+# -*- encoding: utf-8 -*-
+
+import random
+
+import ipaddr
+
+from twisted.internet import defer, reactor
+from twisted.python import usage
+from twisted.internet.endpoints import TCP4ClientEndpoint
+
+from ooni.utils import log
+from ooni.common.http_utils import extractTitle
+from ooni.common.tcp_utils import TCPConnectFactory
+from ooni.errors import failureToString
+
+from ooni.templates import httpt, dnst
+
+# These are taken from https://www.whatsapp.com/cidr.txt
+WHATSAPP_IPV4 = """\
+31.13.64.51/32
+31.13.65.49/32
+31.13.66.49/32
+31.13.67.51/32
+31.13.68.52/32
+31.13.69.240/32
+31.13.70.49/32
+31.13.71.49/32
+31.13.72.52/32
+31.13.73.49/32
+31.13.74.49/32
+31.13.75.52/32
+31.13.76.81/32
+31.13.77.49/32
+31.13.78.53/32
+31.13.79.195/32
+31.13.80.53/32
+31.13.81.53/32
+31.13.82.51/32
+31.13.83.51/32
+31.13.84.51/32
+31.13.85.51/32
+31.13.86.51/32
+31.13.87.51/32
+31.13.88.49/32
+31.13.90.51/32
+31.13.91.51/32
+31.13.92.52/32
+31.13.93.51/32
+31.13.94.52/32
+31.13.95.63/32
+50.22.198.204/30
+50.22.210.32/30
+50.22.210.128/27
+50.22.225.64/27
+50.22.235.248/30
+50.22.240.160/27
+50.23.90.128/27
+50.97.57.128/27
+75.126.39.32/27
+108.168.174.0/27
+108.168.176.192/26
+108.168.177.0/27
+108.168.180.96/27
+108.168.254.65/32
+108.168.255.224/32
+108.168.255.227/32
+157.240.0.53/32
+157.240.3.53/32
+158.85.0.96/27
+158.85.5.192/27
+158.85.46.128/27
+158.85.48.224/27
+158.85.58.0/25
+158.85.61.192/27
+158.85.224.160/27
+158.85.233.32/27
+158.85.249.128/27
+158.85.254.64/27
+169.44.36.0/25
+169.44.57.64/27
+169.44.58.64/27
+169.44.80.0/26
+169.44.82.96/27
+169.44.82.128/27
+169.44.82.192/26
+169.44.83.0/26
+169.44.83.96/27
+169.44.83.128/27
+169.44.83.192/26
+169.44.84.0/24
+169.44.85.64/27
+169.45.71.32/27
+169.45.71.96/27
+169.45.87.128/26
+169.45.169.192/27
+169.45.182.96/27
+169.45.210.64/27
+169.45.214.224/27
+169.45.219.224/27
+169.45.237.192/27
+169.45.238.32/27
+169.45.248.96/27
+169.45.248.160/27
+169.53.29.128/27
+169.53.48.32/27
+169.53.71.224/27
+169.53.250.128/26
+169.53.252.64/27
+169.53.255.64/27
+169.54.2.160/27
+169.54.44.224/27
+169.54.51.32/27
+169.54.55.192/27
+169.54.193.160/27
+169.54.210.0/27
+169.54.222.128/27
+169.55.69.128/26
+169.55.74.32/27
+169.55.126.64/26
+169.55.210.96/27
+169.55.235.160/27
+173.192.162.32/27
+173.192.219.128/27
+173.192.222.160/27
+173.192.231.32/27
+173.193.205.0/27
+173.193.230.96/27
+173.193.230.128/27
+173.193.230.192/27
+173.193.239.0/27
+174.36.208.128/27
+174.36.210.32/27
+174.36.251.192/27
+174.37.199.192/27
+174.37.217.64/27
+174.37.231.64/27
+174.37.243.64/27
+174.37.251.0/27
+179.60.192.51/32
+179.60.193.51/32
+179.60.195.51/32
+184.173.136.64/27
+184.173.147.32/27
+184.173.161.64/32
+184.173.161.160/27
+184.173.173.116/32
+184.173.179.32/27
+185.60.216.53/32
+185.60.218.53/32
+192.155.212.192/27
+198.11.193.182/31
+198.11.251.32/27
+198.23.80.0/27
+208.43.115.192/27
+208.43.117.79/32
+208.43.122.128/27"""
+
+WHATSAPP_IPV6 = """\
+2607:f0d0:1b01:d4::/64
+2607:f0d0:1b02:14d::/64
+2607:f0d0:1b04:32::/64
+2607:f0d0:1b04:bb::/64
+2607:f0d0:1b04:bc::/64
+2607:f0d0:1b06::/64
+2607:f0d0:1b06:4::/64
+2607:f0d0:2102:229::/64
+2607:f0d0:2601:37::/64
+2607:f0d0:3003:1bc::/64
+2607:f0d0:3004:136::/64
+2607:f0d0:3004:174::/64
+2607:f0d0:3005:183::/64
+2607:f0d0:3005:1a3::/64
+2607:f0d0:3006:84::/64
+2607:f0d0:3006:af::/64
+2607:f0d0:3801:38::/64
+2607:f0d0:3802:48::/64
+2a03:2880:f200:c5:face:b00c::167/128
+2a03:2880:f200:1c5:face:b00c::167/128
+2a03:2880:f201:c5:face:b00c::167/128
+2a03:2880:f202:c4:face:b00c::167/128
+2a03:2880:f203:c5:face:b00c::167/128
+2a03:2880:f204:c5:face:b00c::167/128
+2a03:2880:f205:c5:face:b00c::167/128
+2a03:2880:f206:c5:face:b00c::167/128
+2a03:2880:f207:c5:face:b00c::167/128
+2a03:2880:f208:c5:face:b00c::167/128
+2a03:2880:f209:c5:face:b00c::167/128
+2a03:2880:f20a:c5:face:b00c::167/128
+2a03:2880:f20b:c5:face:b00c::167/128
+2a03:2880:f20c:c6:face:b00c::167/128
+2a03:2880:f20d:c5:face:b00c::167/128
+2a03:2880:f20e:c5:face:b00c::167/128
+2a03:2880:f20f:c6:face:b00c::167/128
+2a03:2880:f210:c5:face:b00c::167/128
+2a03:2880:f211:c5:face:b00c::167/128
+2a03:2880:f212:c5:face:b00c::167/128
+2a03:2880:f213:c5:face:b00c::167/128
+2a03:2880:f213:80c5:face:b00c::167/128
+2a03:2880:f214:c5:face:b00c::167/128
+2a03:2880:f215:c5:face:b00c::167/128
+2a03:2880:f216:c5:face:b00c::167/128
+2a03:2880:f217:c5:face:b00c::167/128
+2a03:2880:f218:c3:face:b00c::167/128
+2a03:2880:f219:c5:face:b00c::167/128
+2a03:2880:f21a:c5:face:b00c::167/128
+2a03:2880:f21b:c5:face:b00c::167/128
+2a03:2880:f21c:c5:face:b00c::167/128
+2a03:2880:f21c:80c5:face:b00c::167/128
+2a03:2880:f21f:c5:face:b00c::167/128
+2a03:2880:f221:c5:face:b00c::167/128
+2a03:2880:f222:c5:face:b00c::167/128
+2a03:2880:f223:c5:face:b00c::167/128
+2a03:2880:f225:c4:face:b00c::167/128
+2a03:2880:f226:c6:face:b00c::167/128"""
+
+class DidNotConnect(Exception):
+ pass
+
+class WhatsAppNetwork(object):
+ def __init__(self):
+ self.ipv4_networks = []
+ for ip in WHATSAPP_IPV4.split("\n"):
+ try:
+ self.ipv4_networks.append(ipaddr.IPv4Network(ip))
+ except Exception as exc:
+ log.err("IP is wrong")
+ log.msg(ip)
+ self.ipv6_networks = map(ipaddr.IPv6Network,
+ WHATSAPP_IPV6.split("\n"))
+
+ def contains(self, ip_address):
+ ip = ipaddr.IPAddress(ip_address)
+ if isinstance(ip, ipaddr.IPv4Address):
+ networks = self.ipv4_networks
+ elif isinstance(ip, ipaddr.IPv6Address):
+ networks = self.ipv6_networks
+ else:
+ raise RuntimeError("Should never happen")
+ for network in networks:
+ if network.Contains(ip):
+ return True
+ return False
+
+class UsageOptions(usage.Options):
+ optFlags = [
+ ['all-endpoints', 'e', 'Should we attempt to connect to all whatsapp'
+ ' endpoints?'],
+ ]
+
+class WhatsappTest(httpt.HTTPTest, dnst.DNSTest):
+ name = "Whatsapp"
+ description = ("This test checks to see if the servers used by whatsapp "
+ "messenger are reachable")
+ author = "Arturo Filastò"
+ version = "0.1.0"
+
+ requiresRoot = False
+ requiresTor = False
+ followRedirects = True
+ usageOptions = UsageOptions
+
+ def setUp(self):
+ self.report['registratison_server_failure'] = None
+ self.report['registration_server_status'] = None
+ self.report['whatsapp_web_failure'] = None
+ self.report['whatsapp_web_status'] = None
+
+ self.report['whatsapp_endpoints_status'] = None
+ self.report['whatsapp_endpoints_dns_inconsistent'] = []
+ self.report['whatsapp_endpoints_blocked'] = []
+
+ self.report['tcp_connect'] = []
+
+ @defer.inlineCallbacks
+ def test_registration_server(self):
+ url = 'https://v.whatsapp.net/v2/register'
+ # Ensure I get back:
+ # {"status": "fail", "reason": "missing_param", "param": "code"}
+
+ try:
+ response = yield self.doRequest(url, 'GET')
+ except Exception as exc:
+ failure_string = failureToString(defer.failure.Failure(exc))
+ log.err("Failed to contact the registration server %s" % failure_string)
+ self.report['registratison_server_failure'] = failure_string
+ self.report['registration_server_status'] = 'blocked'
+ defer.returnValue(None)
+
+ log.msg("Successfully connected to registration server!")
+ self.report['registration_server_status'] = 'ok'
+
+ @defer.inlineCallbacks
+ def _test_whatsapp_web(self, url):
+ try:
+ response = yield self.doRequest(url, 'GET')
+ except Exception as exc:
+ failure_string = failureToString(defer.failure.Failure(exc))
+ log.err("Failed to connect to whatsapp web %s" % failure_string)
+ self.report['whatsapp_web_failure'] = failure_string
+ self.report['whatsapp_web_status'] = 'blocked'
+ defer.returnValue(None)
+
+ title = extractTitle(response.body).strip()
+ if title != "WhatsApp Web":
+ self.report['whatsapp_web_status'] = 'blocked'
+
+ @defer.inlineCallbacks
+ def test_whatsapp_web(self):
+ yield self._test_whatsapp_web('https://web.whatsapp.com/')
+ yield self._test_whatsapp_web('http://web.whatsapp.com/')
+ if self.report['whatsapp_web_status'] != 'blocked':
+ self.report['whatsapp_web_status'] = 'ok'
+
+ def _test_connect_to_port(self, address, port):
+ result = {
+ 'ip': address,
+ 'port': port,
+ 'status': {
+ 'success': None,
+ 'failure': None
+ }
+ }
+ point = TCP4ClientEndpoint(reactor, address, port, timeout=10)
+ d = point.connect(TCPConnectFactory())
+ @d.addCallback
+ def cb(p):
+ result['status']['success'] = True
+ result['status']['failure'] = False
+ self.report['tcp_connect'].append(result)
+
+ @d.addErrback
+ def eb(failure):
+ result['status']['success'] = False
+ result['status']['failure'] = failureToString(failure)
+ self.report['tcp_connect'].append(result)
+
+ @defer.inlineCallbacks
+ def _test_connect(self, address):
+ possible_ports = [443, 5222]
+
+ connected = False
+ for port in possible_ports:
+ try:
+ yield self._test_connect_to_port(address, port)
+ connected = False
+ except Exception as exc:
+ pass
+
+ if connected == False:
+ raise DidNotConnect()
+
+ @defer.inlineCallbacks
+ def _test_endpoint(self, hostname, whatsapp_network):
+ log.msg("Testing %s" % hostname)
+ addresses = yield self.performALookup(hostname)
+ consistent = True
+ for address in addresses[:]:
+ try:
+ is_in_whats_app_network = whatsapp_network.contains(address)
+ except ValueError:
+ # This happens when it's not an IP
+ addresses.remove(address)
+ continue
+ if not is_in_whats_app_network:
+ self.report['whatsapp_endpoints_status'] = 'blocked'
+ consistent = False
+
+ if consistent == False:
+ log.msg("%s presents an inconsistent DNS response" % hostname)
+ self.report['whatsapp_endpoints_dns_inconsistent'].append(hostname)
+ defer.returnValue(None)
+
+ dl = []
+ for address in addresses:
+ dl.append(self._test_connect(address))
+ results = yield defer.DeferredList(dl, consumeErrors=True)
+
+ tcp_blocked = False
+ for success, result in results:
+ if success == False:
+ tcp_blocked = True
+
+ if tcp_blocked == True:
+ log.msg("%s is blocked based on TCP")
+ self.report['whatsapp_endpoints_blocked'].append(hostname)
+ self.report['whatsapp_endpoints_status'] = 'blocked'
+ else:
+ self.report['whatsapp_endpoints_status'] = 'ok'
+
+
+ @defer.inlineCallbacks
+ def test_endpoints(self):
+ possible_endpoints = map(lambda x: "e%s.whatsapp.net" % x, range(1, 16))
+ whatsapp_network = WhatsAppNetwork()
+ to_test_endpoints = []
+ if self.localOptions['all-endpoints']:
+ to_test_endpoints += possible_endpoints
+ else:
+ to_test_endpoints += [random.choice(possible_endpoints)]
+ for endpoint in to_test_endpoints:
+ yield self._test_endpoint(endpoint, whatsapp_network)
More information about the tor-commits
mailing list