[tor-commits] [ooni-probe/master] * Removing old tests which have already been ported: dnstamper, echo,
isis at torproject.org
isis at torproject.org
Sat Nov 3 01:24:44 UTC 2012
commit 77c07070e7e8575abc7e6b9fdeed4d7664736ec3
Author: Isis Lovecruft <isis at torproject.org>
Date: Fri Nov 2 16:47:40 2012 +0000
* Removing old tests which have already been ported: dnstamper, echo,
blocking.
---
nettests/core/dnstamper.py | 29 +++--
ooni/plugins/blocking.py | 46 ------
ooni/plugins/dnstamper.py | 338 --------------------------------------------
ooni/plugins/echo.py | 127 -----------------
4 files changed, 19 insertions(+), 521 deletions(-)
diff --git a/nettests/core/dnstamper.py b/nettests/core/dnstamper.py
index b5fcea3..aad2ef3 100644
--- a/nettests/core/dnstamper.py
+++ b/nettests/core/dnstamper.py
@@ -1,6 +1,5 @@
# -*- encoding: utf-8 -*-
#
-#
# dnstamper
# *********
#
@@ -25,16 +24,13 @@ from twisted.names.error import DNSQueryRefusedError
class DNSTamperTest(nettest.TestCase):
name = "DNS tamper"
-
description = "DNS censorship detection test"
version = "0.2"
-
lookupTimeout = [1]
-
requirements = None
+
inputFile = ['file', 'f', None,
'Input file of list of hostnames to attempt to resolve']
-
optParameters = [['controlresolver', 'c', '8.8.8.8',
'Known good DNS server'],
['testresolvers', 't', None,
@@ -43,20 +39,18 @@ class DNSTamperTest(nettest.TestCase):
def setUp(self):
self.report['test_lookups'] = {}
self.report['test_reverse'] = {}
-
self.report['control_lookup'] = []
-
self.report['a_lookups'] = {}
-
self.report['tampering'] = {}
self.test_a_lookups = {}
self.control_a_lookups = []
-
self.control_reverse = None
self.test_reverse = {}
if not self.localOptions['testresolvers']:
+ log.msg("You did not specify a file of DNS servers to test!",
+ "See the '--testresolvers' option.")
self.test_resolvers = ['8.8.8.8']
return
@@ -181,6 +175,14 @@ class DNSTamperTest(nettest.TestCase):
return r
def do_reverse_lookups(self, result):
+ """
+ Take a resolved address in the form "176.139.79.178.in-addr.arpa." and
+ attempt to reverse the domain with both the control and test DNS
+ servers to see if they match.
+
+ :param result:
+ A resolved domain name.
+ """
log.msg("Doing the reverse lookups %s" % self.input)
list_of_ds = []
@@ -209,6 +211,12 @@ class DNSTamperTest(nettest.TestCase):
return dl
def compare_results(self, *arg, **kw):
+ """
+ Take the set intersection of two test result sets. If the intersection
+ is greater than zero (there are matching addresses in both sets) then
+ the no censorship is reported. Else, if no IP addresses match other
+ addresses, then we mark it as a censorship event.
+ """
log.msg("Comparing results for %s" % self.input)
log.msg(self.test_a_lookups)
@@ -222,7 +230,8 @@ class DNSTamperTest(nettest.TestCase):
# Address has not tampered with on DNS server
self.report['tampering'][test] = False
- elif self.control_reverse and set([self.control_reverse]) & set([self.report['test_reverse'][test]]):
+ elif self.control_reverse and set([self.control_reverse]) \
+ & set([self.report['test_reverse'][test]]):
# Further testing has eliminated false positives
self.report['tampering'][test] = 'reverse-match'
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
deleted file mode 100644
index 4dd2db1..0000000
--- a/ooni/plugins/blocking.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-
-from plugoo.assets import Asset
-from plugoo.tests import ITest, OONITest
-
-class BlockingArgs(usage.Options):
- optParameters = [['asset', 'a', None, 'Asset file'],
- ['resume', 'r', 0, 'Resume at this index'],
- ['shit', 'o', None, 'Other arguments']]
-
-class BlockingTest(OONITest):
- implements(IPlugin, ITest)
-
- shortName = "blocking"
- description = "Blocking plugin"
- requirements = None
- options = BlockingArgs
- # Tells this to be blocking.
- blocking = True
-
- def control(self, experiment_result, args):
- print "Experiment Result:", experiment_result
- print "Args", args
- return experiment_result
-
- def experiment(self, args):
- import urllib
- url = 'http://torproject.org/' if not 'asset' in args else args['asset']
- try:
- req = urllib.urlopen(url)
- except:
- return {'error': 'Connection failed!'}
-
- return {'page': req.readlines()}
-
- def load_assets(self):
- if self.local_options and self.local_options['asset']:
- return {'asset': Asset(self.local_options['asset'])}
- else:
- return {}
-
-# We need to instantiate it otherwise getPlugins does not detect it
-# XXX Find a way to load plugins without instantiating them.
-#blocking = BlockingTest(None, None, None)
diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py
deleted file mode 100644
index 40df505..0000000
--- a/ooni/plugins/dnstamper.py
+++ /dev/null
@@ -1,338 +0,0 @@
-# -*- coding: utf-8 -*-
-"""
- dnstamper
- *********
-
- This test resolves DNS for a list of domain names, one per line, in the
- file specified in the ooni-config under the setting "dns_experiment". If
- the file is top-1m.txt, the test will be run using Amazon's list of top
- one million domains. The experimental dns servers to query should
- be specified one per line in assets/dns_servers.txt.
-
- The test reports censorship if the cardinality of the intersection of
- the query result set from the control server and the query result set
- from the experimental server is zero, which is to say, if the two sets
- have no matching results whatsoever.
-
- NOTE: This test frequently results in false positives due to GeoIP-based
- load balancing on major global sites such as google, facebook, and
- youtube, etc.
-
- :author: Isis Lovecruft, Arturo Filastò
- :license: see LICENSE for more details
-
- TODO:
- * Finish porting to twisted
- * Finish the client.Resolver() subclass and test it
- * Use the DNS tests from captiveportal
- * Use plugoo/reports.py for final data
-"""
-
-import os
-
-from twisted.names import client, dns
-from twisted.internet import reactor, defer
-from twisted.internet.error import CannotListenError
-from twisted.internet.protocol import Factory, Protocol
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from zope.interface import implements
-
-from ooni.plugoo.assets import Asset
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.utils import log
-
-class AlexaAsset(Asset):
- """
- Class for parsing the Alexa top-1m.txt as an asset.
- """
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- self = Asset.parse_line(self, line)
- return line.split(',')[1].replace('\n','')
-
-class DNSTamperArgs(usage.Options):
- optParameters = [['hostnames', 'h', None,
- 'Asset file of hostnames to resolve'],
- ['controlresolver', 'c', '8.8.8.8',
- 'Known good DNS server'],
- ['testresolvers', 't', None,
- 'Asset file of DNS servers to test'],
- ['localresolvers', 'l', False,
- 'Also test local servers'],
- ['port', 'p', None,
- 'Local UDP port to send queries over'],
- ['usereverse', 'r', False,
- 'Also try reverse DNS resolves'],
- ['resume', 's', 0,
- 'Resume at this index in the asset file']]
-
-class DNSTamperResolver(client.Resolver):
- """
- Twisted by default issues DNS queries over cryptographically random
- UDP ports to mitigate the Berstein/Kaminsky attack on limited DNS
- Transaction ID numbers.[1][2][3]
-
- This is fine, unless the client has external restrictions which require
- DNS queries to be conducted over UDP port 53. Twisted does not provide
- an easy way to change this, ergo subclassing client.Resolver.[4] It
- would perhaps be wise to patch twisted.names.client and request a merge
- into upstream.
-
- [1] https://twistedmatrix.com/trac/ticket/3342
- [2] http://blog.netherlabs.nl/articles/2008/07/09/ \
- some-thoughts-on-the-recent-dns-vulnerability
- [3] http://www.blackhat.com/presentations/bh-dc-09/Kaminsky/ \
- BlackHat-DC-09-Kaminsky-DNS-Critical-Infrastructure.pdf
- [4] http://comments.gmane.org/gmane.comp.python.twisted/22794
- """
- def __init__(self):
- super(DNSTamperResolver, self).__init__(self, resolv, servers,
- timeout, reactor)
- #client.Resolver.__init__(self)
-
- if self.local_options['port']:
- self.port = self.local_options['port']
- else:
- self.port = '53'
-
- def _connectedProtocol(self):
- """
- Return a new DNSDatagramProtocol bound to a specific port
- rather than the default cryptographically-random port.
- """
- if 'protocol' in self.__dict__:
- return self.protocol
- proto = dns.DNSDatagramProtocol(self)
-
- ## XXX We may need to remove the while loop, which was
- ## originally implemented to safeguard against attempts to
- ## bind to the same random port twice...but then the code
- ## would be blocking...
- while True:
- try:
- self._reactor.listenUDP(self.port, proto)
- except error.CannotListenError:
- pass
- else:
- return proto
-
-class DNSTamperTest(OONITest):
- """
- XXX fill me in
- """
- implements(IPlugin, ITest)
-
- shortName = "dnstamper"
- description = "DNS censorship detection test"
- requirements = None
- options = DNSTamperArgs
- blocking = False
-
- def __init__(self, local_options, global_options,
- report, ooninet=None, reactor=None):
- super(DNSTamperTest, self).__init__(local_options, global_options,
- report, ooninet, reactor)
-
- def __repr__(self):
- represent = "DNSTamperTest(OONITest): local_options=%r, " \
- "global_options=%r, assets=%r" % (self.local_options,
- self.global_options,
- self.assets)
- return represent
-
- def initialize(self):
- if self.local_options:
- ## client.createResolver() turns 'None' into '/etc/resolv.conf' on
- ## posix systems, ignored on Windows.
- if self.local_options['localresolvers']:
- self.resolvconf = None
- else:
- self.resolvconf = ''
-
- def load_assets(self):
- assets = {}
-
- #default_hostnames = ['baidu.com', 'torrentz.eu', 'twitter.com',
- # 'ooni.nu', 'google.com', 'torproject.org']
- #default_resolvers = ['209.244.0.3', '208.67.222.222']
-
- def asset_file(asset_option):
- return self.local_options[asset_option]
-
- def list_to_asset(list_):
- def next(list_):
- host = list_.pop()
- if host is not None:
- yield str(host)
- while len(list_) > 0:
- next(list_)
-
- if self.local_options:
- if asset_file('hostnames'):
- ## The default filename for the Alexa Top 1 Million:
- if asset_file('hostnames') == 'top-1m.txt':
- assets.update({'hostnames':
- AlexaAsset(asset_file('hostnames'))})
- else:
- assets.update({'hostnames':
- Asset(asset_file('hostnames'))})
- else:
- log.msg("Error! We need an asset file containing the " +
- "hostnames that we should test DNS with! Please use " +
- "the '-h' option. Using pre-defined hostnames...")
-
- if asset_file('testresolvers'):
- assets.update({'testresolvers':
- Asset(asset_file('testresolvers'))})
-
- return assets
-
- def lookup(self, hostname, resolver):
- """
- Resolves a hostname through a DNS nameserver to the corresponding IP
- addresses.
- """
- def got_result(result, hostname, resolver):
- log.msg('Resolved %s through %s to %s'
- % (hostname, resolver, result))
- report = {'resolved': True,
- 'domain': hostname,
- 'nameserver': resolver,
- 'address': result }
- log.msg(report)
- return result
-
- def got_error(err, hostname, resolver):
- log.msg(err.printTraceback())
- report = {'resolved': False,
- 'domain': hostname,
- 'nameserver': resolver,
- 'address': err }
- log.msg(report)
- return err
-
- res = client.createResolver(resolvconf=self.resolvconf,
- servers=[(resolver, 53)])
-
- ## XXX should we do self.d.addCallback(resHostByName, hostname)?
- #d = res.getHostByName(hostname)
- #d.addCallbacks(got_result, got_error)
-
- #d = defer.Deferred()
- #d.addCallback(res.getHostByName, hostname)
-
- #d = res.getHostByName(hostname)
- #d.addCallback(got_result, result, hostname, resolver)
- #d.addErrback(got_error, err, hostname, resolver)
-
- res.addCallback(getHostByName, hostname)
- res.addCallback(got_result, result, hostname, resolver)
- res.addErrback(got_error, err, hostname, resolver)
-
- if self.local_options['usereverse']:
- #d.addCallback(self.reverse_lookup, result, resolver)
- #d.addErrback(log.msg(err.printTraceback()))
-
- #d.addCallback(self.reverse_lookup, result, resolver)
- #d.addErrback(log.msg(err.printTraceback()))
-
- res.addCallback(self.reverse_lookup, result, resolver)
- res.addErraback(log.msg(err.printTraceback()))
-
- return res
-
- def reverse_lookup(self, address, resolver):
- """
- Attempt to do a reverse DNS lookup to determine if the control and exp
- sets from a positive result resolve to the same domain, in order to
- remove false positives due to GeoIP load balancing.
- """
- res = client.createResolver(resolvconf=self.resolvconf,
- servers=[(resolver, 53)])
- ptr = '.'.join(addr.split('.')[::-1]) + '.in-addr.arpa'
- reverse = res.lookupPointer(ptr)
- reverse.addCallback(lambda (address, auth, add):
- util.println(address[0].payload.name))
- reverse.addErrback(log.err)
-
- ## XXX do we need to stop the reactor?
- #d.addBoth(lambda r: reactor.stop())
-
- return reverse
-
- def experiment(self, args):
- """
- Compares the lookup() sets of the control and experiment groups.
- """
- for hostname in args:
- for testresolver in self.assets['testresolvers']:
- #addressd = defer.Deferred()
- #addressd.addCallback(self.lookup, hostname, testresolver)
- #addressd.addErrback(log.err)
-
- self.d.addCallback(self.lookup, hostname, testresolver)
- self.d.addErrback(log.err)
-
- #addressd = self.lookup(hostname, testresolver)
-
- #self.d.addCallback(self.lookup, hostname, testserver)
-
- print "%s" % type(addressd)
-
- return self.d
-
- def control(self, experiment_result, args):
- print "EXPERIMENT RESULT IS %s" % experiment_result
- (exp_address, hostname, testserver, exp_reversed) = experiment_result
- control_server = self.local_options['controlserver']
- ctrl_address = self.lookup(hostname, control_server)
-
- ## XXX getHostByName() appears to be returning only one IP...
-
- if len(set(exp_address) & set(ctrl_address)) > 0:
- log.msg("Address %s has not tampered with on DNS server %s"
- % (hostname, test_server))
- return {'hostname': hostname,
- 'test-nameserver': test_server,
- 'test-address': exp_address,
- 'control-nameserver': control_server,
- 'control-address': ctrl_address,
- 'tampering-detected': False}
- else:
- log.msg("Address %s has possibly been tampered on %s:"
- % (hostname, test_server))
- log.msg("DNS resolution through testserver %s yeilds: %s"
- % (test_server, exp_address))
- log.msg("However, DNS resolution through controlserver %s yeilds: %s"
- % (control_server, ctrl_address))
-
- if self.local_options['usereverse']:
- ctrl_reversed = self.reverse_lookup(experiment_result, control_server)
- if len(set(ctrl_reversed) & set(exp_reversed)) > 0:
- log.msg("Further testing has eliminated false positives")
- else:
- log.msg("Reverse DNS on the results returned by %s returned:"
- % (test_server))
- log.msg("%s" % exp_reversed)
- log.msg("which does not match the expected domainname: %s"
- % ctrl_reversed)
- return {'hostname': hostname,
- 'test-nameserver': test_server,
- 'test-address': exp_address,
- 'test-reversed': exp_reversed,
- 'control-nameserver': control_server,
- 'control-address': ctrl_address,
- 'control-reversed': ctrl_reversed,
- 'tampering-detected': True}
- else:
- return {'hostname': hostname,
- 'test-nameserver': test_server,
- 'test-address': exp_address,
- 'control-nameserver': control_server,
- 'control-address': ctrl_address,
- 'tampering-detected': False}
-
-#dnstamper = DNSTamperTest(None, None, None)
diff --git a/ooni/plugins/echo.py b/ooni/plugins/echo.py
deleted file mode 100644
index bc1b2a8..0000000
--- a/ooni/plugins/echo.py
+++ /dev/null
@@ -1,127 +0,0 @@
-#!/usr/bin/env python
-# -*- encoding: utf-8 -*-
-#
-# +---------+
-# | echo.py |
-# +---------+
-# A simply ICMP-8 ping test.
-#
-# :author: Isis Lovecruft
-# :version: 0.1.0-pre-alpha
-# :license: (c) 2012 Isis Lovecruft
-# see attached LICENCE file
-#
-
-import os
-import sys
-
-from twisted.plugin import IPlugin
-from twisted.python import usage
-from zope.interface import implements
-
-from lib import txscapy
-from utils import log
-from plugoo.assets import Asset
-from plugoo.interface import ITest
-from protocols.scapyproto import ScapyTest
-
-class EchoOptions(usage.Options):
- optParameters = [
- ['interface', 'i', None, 'Network interface to use'],
- ['destination', 'd', None, 'File of hosts to ping'],
- ['count', 'c', 5, 'Number of packets to send', int],
- ['size', 's', 56, 'Number of bytes to send in ICMP data field', int],
- ['ttl', 't', 25, 'Set the IP Time to Live', int],
- ]
- optFlags = []
-
-class EchoAsset(Asset):
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
- def parse_line(self, line):
- if line.startswith('#'):
- return
- else:
- return line.replace('\n', '')
-
-class EchoTest(ScapyTest):
- implements(IPlugin, ITest)
-
- shortName = 'echo'
- description = 'A simple ICMP-8 test to check if a host is reachable'
- options = EchoOptions
- requirements = None
- blocking = False
-
- pcap_file = 'echo.pcap'
- receive = True
-
- def initialize(self):
- self.request = {}
- self.response = {}
-
- if self.local_options:
-
- options = self.local_options
-
- if options['interface']:
- self.interface = options['interface']
-
- if options['count']:
- ## there's a Counter() somewhere, use it
- self.count = options['count']
-
- if options['size']:
- self.size = options['size']
-
- if options['ttl']:
- self.ttl = options['ttl']
-
- def load_assets(self):
- assets = {}
- option = self.local_options
-
- if option and option['destination']:
-
- try:
- from scapy.all import IP
- except:
- log.err()
-
- if os.path.isfile(option['destination']):
- with open(option['destination']) as hosts:
- for line in hosts.readlines():
- assets.update({'host': EchoAsset(line)})
- else:
- while type(options['destination']) is str:
- try:
- IP(options['destination'])
- except:
- log.err()
- break
- assets.update({'host': options['destination']})
- else:
- log.msg("Couldn't understand destination option...")
- log.msg("Give one IPv4 address, or a file with one address per line.")
- return assets
-
- def experiment(self, args):
- if len(args) == 0:
- log.err("Error: We're Echo, not Narcissus!")
- log.err(" Provide a list of hosts to ping...")
- d = sys.exit(1)
- return d
-
- ## XXX v4 / v6
- from scapy.all import ICMP, IP, sr
- ping = sr(IP(dst=args)/ICMP())
- if ping:
- self.response.update(ping.show())
- else:
- log.msg('No response received from %s' % args)
-
- def control(self, *args):
- pass
-
-echo = EchoTest(None, None, None)
More information about the tor-commits
mailing list