[tor-commits] [ooni-probe/master] * Modified the ValueChecker class to have static methods.
isis at torproject.org
isis at torproject.org
Thu Oct 4 14:41:15 UTC 2012
commit d448f256ee5993237d9182164ba0efae808c18b6
Author: Isis Lovecruft <isis at torproject.org>
Date: Fri Sep 28 04:58:00 2012 +0000
* Modified the ValueChecker class to have static methods.
* Switched to dictionaries for self.relays and self.bridges so that all
methods can access various values without having to constantly change
parameters.
* Fixed a couple assertions.
---
ooni/plugins/bridget.py | 352 ++++++++++++++++++++++------------------------
ooni/utils/config.py | 9 +-
ooni/utils/onion.py | 55 +++++---
3 files changed, 210 insertions(+), 206 deletions(-)
diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py
index c749b90..208b4c3 100644
--- a/ooni/plugins/bridget.py
+++ b/ooni/plugins/bridget.py
@@ -12,25 +12,26 @@
# :licence: see included LICENSE
# :version: 0.1.0-alpha
-from __future__ import with_statement
-from functools import partial
-from random import randint
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from twisted.internet import defer, error, reactor
-from zope.interface import implements
-
-from ooni.utils import log, date
-from ooni.utils.config import ValueChecker
-from ooni.utils.onion import parse_data_dir
-from ooni.utils.onion import TxtorconImportError
-from ooni.utils.onion import PTNoBridgesException, PTNotFoundException
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset, MissingAssetException
+from __future__ import with_statement
+from functools import partial
+from random import randint
import os
import sys
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from twisted.internet import defer, error, reactor
+from zope.interface import implements
+
+from ooni.utils import log, date
+from ooni.utils.config import ValueChecker
+from ooni.utils.onion import parse_data_dir
+from ooni.utils.onion import TxtorconImportError
+from ooni.utils.onion import PTNoBridgesException, PTNotFoundException
+from ooni.plugoo.tests import ITest, OONITest
+from ooni.plugoo.assets import Asset, MissingAssetException
+
class RandomPortException(Exception):
"""Raised when using a random port conflicts with configured ports."""
@@ -40,12 +41,9 @@ class RandomPortException(Exception):
class BridgetArgs(usage.Options):
"""Commandline options."""
- global vc
- vc = ValueChecker
-
allowed = "Port to use for Tor's %s, must be between 1024 and 65535."
- sock_check = vc(allowed % "SocksPort").port_check
- ctrl_check = vc(allowed % "ControlPort").port_check
+ sock_check = ValueChecker(allowed % "SocksPort").port_check
+ ctrl_check = ValueChecker(allowed % "ControlPort").port_check
optParameters = [
['bridges', 'b', None,
@@ -69,7 +67,7 @@ class BridgetArgs(usage.Options):
raise MissingAssetException(
"Bridget can't run without bridges or relays to test!")
if self['transport']:
- vc().uid_check(
+ ValueChecker.uid_check(
"Can't run bridget as root with pluggable transports!")
if not self['bridges']:
raise PTNoBridgesException
@@ -77,9 +75,9 @@ class BridgetArgs(usage.Options):
if self['random']:
raise RandomPortException
if self['datadir']:
- vc().dir_check(self['datadir'])
+ ValueChecker.dir_check(self['datadir'])
if self['torpath']:
- vc().file_check(self['torpath'])
+ ValueChecker.file_check(self['torpath'])
class BridgetAsset(Asset):
"""Class for parsing bridget Assets ignoring commented out lines."""
@@ -126,29 +124,11 @@ class BridgetTest(OONITest):
running, so we need to deal with most of the TorConfig() only once,
before the experiment runs.
"""
- self.d = defer.Deferred()
-
self.socks_port = 9049
self.control_port = 9052
self.circuit_timeout = 90
self.tor_binary = '/usr/sbin/tor'
self.data_directory = None
- self.use_pt = False
- self.pt_type = None
-
- ## XXX we should do report['bridges_up'].append(self.current_bridge)
- self.bridges, self.bridges_up, self.bridges_down = ([] for i in range(3))
- self.bridges_remaining = lambda: len(self.bridges)
- self.bridges_down_count = lambda: len(self.bridges_down)
- self.current_bridge = None
-
- self.relays, self.relays_up, self.relays_down = ([] for i in range(3))
- self.relays_remaining = lambda: len(self.relays)
- self.relays_down_count = lambda: len(self.relays_down)
- self.current_relay = None
-
- ## Make sure we don't use self.load_assets() for now:
- self.assets = {}
def __make_asset_list__(opt, lst):
log.msg("Loading information from %s ..." % opt)
@@ -159,52 +139,71 @@ class BridgetTest(OONITest):
else:
lst.append(line.replace('\n',''))
+ def __count_remaining__(which):
+ total, reach, unreach = map(lambda x: which[x],
+ ['all', 'reachable', 'unreachable'])
+ count = len(total) - reach() - unreach()
+ return count
+
+ ## XXX should we do report['bridges_up'].append(self.bridges['current'])
+ self.bridges = {}
+ self.bridges['all'], self.bridges['up'], self.bridges['down'] = \
+ ([] for i in range(3))
+ self.bridges['reachable'] = lambda: len(self.bridges['up'])
+ self.bridges['unreachable'] = lambda: len(self.bridges['down'])
+ self.bridges['remaining'] = lambda: __count_remaining__(self.bridges)
+ self.bridges['current'] = None
+ self.bridges['pt_type'] = None
+ self.bridges['use_pt'] = False
+
+ self.relays = {}
+ self.relays['all'], self.relays['up'], self.relays['down'] = \
+ ([] for i in range(3))
+ self.relays['reachable'] = lambda: len(self.relays['up'])
+ self.relays['unreachable'] = lambda: len(self.relays['down'])
+ self.relays['remaining'] = lambda: __count_remaining__(self.relays)
+ self.relays['current'] = None
+
if self.local_options:
try:
from ooni.lib.txtorcon import TorConfig
except ImportError:
raise TxtorconImportError
-
- options = self.local_options
- config = self.config = TorConfig()
+ else:
+ self.config = TorConfig()
+ finally:
+ options = self.local_options
if options['bridges']:
self.config.UseBridges = 1
- __make_asset_list__(options['bridges'], self.bridges)
-
+ __make_asset_list__(options['bridges'], self.bridges['all'])
if options['relays']:
- ## first hop must be in TorState().guards to build circuits
+ ## first hop must be in TorState().guards
self.config.EntryNodes = ','.join(relay_list)
- __make_asset_list__(options['relays'], self.relays)
-
+ __make_asset_list__(options['relays'], self.relays['all'])
if options['socks']:
self.socks_port = options['socks']
if options['control']:
self.control_port = options['control']
-
if options['random']:
log.msg("Using randomized ControlPort and SocksPort ...")
self.socks_port = randint(1024, 2**16)
self.control_port = randint(1024, 2**16)
-
if options['torpath']:
self.tor_binary = options['torpath']
-
if options['datadir']:
self.data_directory = parse_data_dir(options['datadir'])
- else:
- self.data_directory = None
-
if options['transport']:
- self.use_pt = True
- log.msg("Using ClientTransportPlugin %s"
- % options['transport'])
- [self.pt_type, pt_exec] = options['transport'].split(' ', 1)
-
## ClientTransportPlugin transport exec pathtobinary [options]
## XXX we need a better way to deal with all PTs
- if self.pt_type == "obfs2":
- self.config.ClientTransportPlugin = self.pt_type+" "+pt_exec
+ log.msg("Using ClientTransportPlugin %s" % options['transport'])
+ self.bridges['use_pt'] = True
+ [self.bridges['pt_type'], pt_exec] = \
+ options['transport'].split(' ', 1)
+
+ if self.bridges['pt_type'] == "obfs2":
+ self.config.ClientTransportPlugin = \
+ self.bridges['pt_type'] + " " + pt_exec
else:
raise PTNotFoundException
@@ -212,11 +211,21 @@ class BridgetTest(OONITest):
self.config.ControlPort = self.control_port
self.config.CookieAuthentication = 1
- def load_assets(self):
+ def __load_assets__(self):
"""
Load bridges and/or relays from files given in user options. Bridges
should be given in the form IP:ORport. We don't want to load these as
assets, because it's inefficient to start a Tor process for each one.
+
+ We cannot use the Asset model, because that model calls
+ self.experiment() with the current Assets, which would be one relay
+ and one bridge, then it gives the defer.Deferred returned from
+ self.experiment() to self.control(), which means that, for each
+ (bridge, relay) pair, experiment gets called again, which instantiates
+ an additional Tor process that attempts to bind to the same
+ ports. Thus, additionally instantiated Tor processes return with
+ RuntimeErrors, which break the final defer.chainDeferred.callback(),
+ sending it into the errback chain.
"""
assets = {}
if self.local_options:
@@ -229,50 +238,40 @@ class BridgetTest(OONITest):
return assets
def experiment(self, args):
- """
- We cannot use the Asset model, because that model calls
- self.experiment() with the current Assets, which would be one relay
- and one bridge, then it gives the defer.Deferred returned from
- self.experiment() to self.control(), which means that, for each
- (bridge, relay) pair, experiment gets called again, which instantiates
- an additional Tor process that attempts to bind to the same
- ports. Thus, additionally instantiated Tor processes return with
- RuntimeErrors, which break the final defer.chainDeferred.callback(),
- sending it into the errback chain.
-
- if bridges:
- 1. configure first bridge line
- 2a. configure data_dir, if it doesn't exist
- 2b. write torrc to a tempfile in data_dir
- 3. start tor } if any of these
- 4. remove bridges which are public relays } fail, add current
- 5. SIGHUP for each bridge } bridge to unreach-
- } able bridges.
- if relays:
- 1a. configure the data_dir, if it doesn't exist
- 1b. write torrc to a tempfile in data_dir
- 2. start tor
- 3. remove any of our relays which are already part of current
- circuits
- 4a. attach CustomCircuit() to self.state
- 4b. RELAY_EXTEND for each relay } if this fails, add
- } current relay to list
- } of unreachable relays
- 5.
- if bridges and relays:
- 1. configure first bridge line
- 2a. configure data_dir if it doesn't exist
- 2b. write torrc to a tempfile in data_dir
- 3. start tor
- 4. remove bridges which are public relays
- 5. remove any of our relays which are already part of current
- circuits
- 6a. attach CustomCircuit() to self.state
- 6b. for each bridge, build three circuits, with three
- relays each
- 6c. RELAY_EXTEND for each relay } if this fails, add
- } current relay to list
- } of unreachable relays
+ """
+ if bridges:
+ 1. configure first bridge line
+ 2a. configure data_dir, if it doesn't exist
+ 2b. write torrc to a tempfile in data_dir
+ 3. start tor } if any of these
+ 4. remove bridges which are public relays } fail, add current
+ 5. SIGHUP for each bridge } bridge to unreach-
+ } able bridges.
+ if relays:
+ 1a. configure the data_dir, if it doesn't exist
+ 1b. write torrc to a tempfile in data_dir
+ 2. start tor
+ 3. remove any of our relays which are already part of current
+ circuits
+ 4a. attach CustomCircuit() to self.state
+ 4b. RELAY_EXTEND for each relay } if this fails, add
+ } current relay to list
+ } of unreachable relays
+ 5.
+ if bridges and relays:
+ 1. configure first bridge line
+ 2a. configure data_dir if it doesn't exist
+ 2b. write torrc to a tempfile in data_dir
+ 3. start tor
+ 4. remove bridges which are public relays
+ 5. remove any of our relays which are already part of current
+ circuits
+ 6a. attach CustomCircuit() to self.state
+ 6b. for each bridge, build three circuits, with three
+ relays each
+ 6c. RELAY_EXTEND for each relay } if this fails, add
+ } current relay to list
+ } of unreachable relays
:param args:
The :class:`BridgetAsset` line currently being used. Except that it
@@ -290,59 +289,52 @@ class BridgetTest(OONITest):
log.err(tie)
sys.exit()
+ def reconfigure_done(state, bridges):
+ log.msg("Reconfiguring with 'Bridge %s' successful"
+ % bridges['current'])
+ bridges['up'].append(bridges['current'])
+ return state
+
+ def reconfigure_fail(state, bridges):
+ log.msg("Reconfiguring TorConfig with parameters %s failed"
+ % state)
+ bridges['down'].append(bridges['current'])
+ return state
+
@defer.inlineCallbacks
- def reconfigure_bridge(state, bridge, use_pt=False, pt_type=None):
+ def reconfigure_bridge(state, bridges):
"""
Rewrite the Bridge line in our torrc. If use of pluggable
transports was specified, rewrite the line as:
Bridge <transport_type> <ip>:<orport>
Otherwise, rewrite in the standard form.
"""
- log.msg("Current Bridge: %s" % bridge)
+ log.msg("Current Bridge: %s" % bridges['current'])
+ log.msg("We now have %d bridges remaining to test..."
+ % bridges['remaining']())
try:
- if use_pt is False:
- reset_tor = yield state.protocol.set_conf('Bridge',
- bridge)
- elif use_pt and pt_type is not None:
- reset_tor = yield state.protocol.set_conf(
- 'Bridge', pt_type +' '+ bridge)
+ if bridges['use_pt'] is False:
+ controller_response = yield state.protocol.set_conf(
+ 'Bridge', bridges['current'])
+ elif bridges['use_pt'] and bridges['pt_type'] is not None:
+ controller_reponse = yield state.protocol.set_conf(
+ 'Bridge', bridges['pt_type'] +' '+ bridges['current'])
else:
raise PTNotFoundException
- controller_response = reset_tor.callback
- controller_response.addCallback(reconfigure_done,
- bridge,
- reachable)
- controller_response.addErrback(reconfigure_fail,
- bridge,
- reachable)
-
- #if not controller_response:
- # defer.returnValue((state.callback, None))
- #else:
- # defer.returnValue((state.callback, controller_response))
if controller_response == 'OK':
- defer.returnValue((state, controller_response))
+ finish = reconfigure_done(state, bridges)
else:
- log.msg("TorControlProtocol responded with error:\n%s"
- % controller_response)
- defer.returnValue((state.callback, None))
-
- except Exception, e:
- log.msg("Reconfiguring torrc with Bridge line %s failed:\n%s"
- % (bridge, e))
- defer.returnValue((state.callback, e))
+ log.err("SETCONF for %s responded with error:\n %s"
+ % (bridges['current'], controller_response))
+ finish = reconfigure_fail(state, bridges)
- def reconfigure_done(state, bridge, reachable):
- log.msg("Reconfiguring with 'Bridge %s' successful" % bridge)
- reachable.append(bridge)
- return state
+ defer.returnValue(finish)
- def reconfigure_fail(state, bridge, unreachable):
- log.msg("Reconfiguring TorConfig with parameters %s failed"
- % state)
- unreachable.append(bridge)
- return state
+ except Exception, e:
+ log.err("Reconfiguring torrc with Bridge line %s failed:\n%s"
+ % (bridges['current'], e))
+ defer.returnValue(None)
def attacher_extend_circuit(attacher, deferred, router):
## XXX todo write me
@@ -373,25 +365,21 @@ class BridgetTest(OONITest):
log.err("Attaching custom circuit builder failed: %s" % state)
- ## Start the experiment
- log.msg("Bridget: initiating test ... ")
- all_of_the_bridges = self.bridges
- all_of_the_relays = self.relays ## Local copy of orginal lists
+ log.msg("Bridget: initiating test ... ") ## Start the experiment
+ all_of_the_bridges = self.bridges['all']
+ all_of_the_relays = self.relays['all'] ## Local copy of orginal lists
- if self.bridges_remaining() >= 1 and not 'Bridge' in self.config.config:
- ## XXX we should do self.bridges[0] + self.bridges[1:]
- initial_bridge = all_of_the_bridges.pop()
- self.config.Bridge = initial_bridge
- self.config.save() ## avoid starting several processes
+ if self.bridges['remaining']() >= 1 and not 'Bridge' in self.config.config:
+ self.bridges['current'] = self.bridges['all'][0]
+ self.config.Bridge = self.bridges['current']
+ ## avoid starting several
+ self.config.save() ## processes
assert self.config.config.has_key('Bridge'), "NO BRIDGE"
- state = start_tor(self.reactor, self.config,
- self.control_port, self.tor_binary,
- self.data_directory).addCallbacks(
- setup_done,
- errback=setup_fail)
- state.addCallback(remove_public_relays,
- self.bridges)
+ state = start_tor(reactor, self.config, self.control_port,
+ self.tor_binary, self.data_directory)
+ state.addCallbacks(setup_done, setup_fail)
+ state.addCallback(remove_public_relays, self.bridges)
#controller = singleton_semaphore(bootstrap)
#controller = x().addCallback(singleton_semaphore, tor)
@@ -404,17 +392,12 @@ class BridgetTest(OONITest):
log.debug("Current callbacks on TorState():\n%s" % state.callbacks)
log.debug("TorState():\n%s" % state)
- if self.bridges_remaining() > 0:
+ if self.bridges['remaining']() >= 2:
all = []
- for bridge in self.bridges:
- #self.current_bridge = bridge
+ for bridge in self.bridges['all'][1:]:
+ self.bridges['current'] = bridge
new = defer.Deferred()
- new.addCallback(reconfigure_bridge, state, bridge,
- self.bridges_remaining(),
- self.bridges_up,
- self.bridges_down,
- use_pt=self.use_pt,
- pt_type=self.pt_type)
+ new.addCallback(reconfigure_bridge, state, self.bridges)
all.append(new)
#state.chainDeferred(defer.DeferredList(all))
@@ -426,38 +409,39 @@ class BridgetTest(OONITest):
# % controller.callbacks)
state.chainDeferred(check_remaining)
- if self.relays_remaining() > 0:
- while self.relays_remaining() >= 3:
+ if self.relays['remaining']() > 0:
+ while self.relays['remaining']() >= 3:
#path = list(self.relays.pop() for i in range(3))
#log.msg("Trying path %s" % '->'.join(map(lambda node:
# node, path)))
- self.current_relay = self.relays.pop()
+ self.relays['current'] = self.relays['all'].pop()
for circ in state.circuits.values():
for node in circ.path:
- if node == self.current_relay:
- self.relays_up.append(self.current_relay)
+ if node == self.relays['current']:
+ self.relays['up'].append(self.relays['current'])
if len(circ.path) < 3:
try:
ext = attacher_extend_circuit(state.attacher, circ,
- self.current_relay)
+ self.relays['current'])
ext.addCallback(attacher_extend_circuit_done,
state.attacher, circ,
- self.current_relay)
+ self.relays['current'])
except Exception, e:
- log.msg("Extend circuit failed: %s" % e)
+ log.err("Extend circuit failed: %s" % e)
else:
continue
- state.callback(all)
+ #state.callback(all)
self.reactor.run()
- #return state
+ return state
def startTest(self, args):
self.start_time = date.now()
- log.msg("Starting %s" % self.shortName)
- self.do_science = self.experiment(args)
- self.do_science.addCallback(self.finished).addErrback(log.err)
- return self.do_science
+ self.laboratory = defer.Deferred()
+ self.laboratory.addCallbacks(self.experiment, errback=log.err)
+ self.laboratory.addCallbacks(self.finished, errback=log.err)
+ self.laboratory.callback(args)
+ return self.laboratory
## So that getPlugins() can register the Test:
bridget = BridgetTest(None, None, None)
diff --git a/ooni/utils/config.py b/ooni/utils/config.py
index 7c28f33..e429ef6 100644
--- a/ooni/utils/config.py
+++ b/ooni/utils/config.py
@@ -85,7 +85,8 @@ class ValueChecker(object):
raise ValueError("Port out of range")
log.err()
- def uid_check(self, error_message):
+ @staticmethod
+ def uid_check(error_message):
"""
Check that we're not root. If we are, setuid(1000) to normal user if
we're running on a posix-based system, and if we're on Windows just
@@ -105,13 +106,15 @@ class ValueChecker(object):
else:
sys.exit(0)
- def dir_check(self, d):
+ @staticmethod
+ def dir_check(d):
"""Check that the given directory exists."""
if not os.path.isdir(d):
raise ValueError("%s doesn't exist, or has wrong permissions"
% d)
- def file_check(self, f):
+ @staticmethod
+ def file_check(f):
"""Check that the given file exists."""
if not os.path.isfile(f):
raise ValueError("%s does not exist, or has wrong permissions"
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
index 82f013f..6f6b355 100644
--- a/ooni/utils/onion.py
+++ b/ooni/utils/onion.py
@@ -68,17 +68,31 @@ def parse_data_dir(data_dir):
:return:
The absolute path of :param:data_dir.
"""
- import os
+ from os import path, getcwd
+ import sys
+
+ try:
+ assert isinstance(data_dir, str), \
+ "Parameter type(data_dir) must be str"
+ except AssertionError, ae:
+ log.err(ae)
if data_dir.startswith('~'):
- data_dir = os.path.expanduser(data_dir)
+ data_dir = path.expanduser(data_dir)
elif data_dir.startswith('/'):
- data_dir = os.path.join(os.getcwd(), data_dir)
+ data_dir = path.join(getcwd(), data_dir)
elif data_dir.startswith('./'):
- data_dir = os.path.abspath(data_dir)
+ data_dir = path.abspath(data_dir)
+ else:
+ data_dir = path.join(getcwd(), data_dir)
+
+ try:
+ assert path.isdir(data_dir), "Could not find %s" % data_dir
+ except AssertionError, ae:
+ log.err(ae)
+ sys.exit(1)
else:
- data_dir = os.path.join(os.getcwd(), data_dir)
- return data_dir
+ return data_dir
def write_torrc(conf, data_dir=None):
"""
@@ -137,11 +151,10 @@ def remove_node_from_list(node, list):
if item.startswith(node): ## due to the :<port>.
try:
log.msg("Removing %s because it is a public relay" % node)
- list.remove(line)
+ list.remove(item)
except ValueError, ve:
log.err(ve)
- at defer.inlineCallbacks
def remove_public_relays(state, bridges):
"""
Remove bridges from our bridge list which are also listed as public
@@ -151,25 +164,26 @@ def remove_public_relays(state, bridges):
XXX Does state.router.values() have all of the relays in the consensus, or
just the ones we know about so far?
+
+ XXX FIXME: There is a problem in that Tor needs a Bridge line to already be
+ configured in order to bootstrap. However, after bootstrapping, we grab the
+ microdescriptors of all the relays and check if any of our bridges are
+ listed as public relays. Because of this, the first bridge does not get
+ checked for being a relay.
"""
- IPs = map(lambda addr: addr.split(':',1)[0], bridges)
+ IPs = map(lambda addr: addr.split(':',1)[0], bridges['all'])
both = set(state.routers.values()).intersection(IPs)
if len(both) > 0:
try:
- updated = yield map(lambda node: remove_node_from_list(node),
- both)
+ updated = map(lambda node: remove_node_from_list(node), both)
if not updated:
- ## XXX do these need to be state.callback?
defer.returnValue(state)
else:
defer.returnValue(state)
except Exception, e:
- log.msg("Removing public relays from bridge list failed:\n%s"
- % both)
- log.err(e)
- except ValueError, ve:
- log.err(ve)
+ log.err("Removing public relays %s from bridge list failed:\n%s"
+ % (both, e))
#@defer.inlineCallbacks
def start_tor(reactor, config, control_port, tor_binary, data_dir,
@@ -259,6 +273,7 @@ def start_tor(reactor, config, control_port, tor_binary, data_dir,
transport = reactor.spawnProcess(process_protocol,
tor_binary,
args=(tor_binary,'-f',torrc),
+ env={'HOME':data_dir},
path=data_dir)
transport.closeStdin()
except RuntimeError, e:
@@ -337,8 +352,10 @@ class CustomCircuit(CircuitListenerMixin):
for i in range(2))
path = [first, middle, last]
else:
- assert type(path) is list, "Circuit path must be a list of relays!"
- assert len(path) >= 3, "Circuits must be at least three hops!"
+ assert isinstance(path, list), \
+ "Circuit path must be a list of relays!"
+ assert len(path) >= 3, \
+ "Circuit path must be at least three hops!"
log.msg("Requesting a circuit: %s"
% '->'.join(map(lambda node: node, path)))
More information about the tor-commits
mailing list