[tor-commits] [ooni-probe/master] Implement downloading of input files based on URL in deck file

[art at torproject.org]

commit f62f822b926eab78153c3081ced38b1075b16e10
Author: Arturo Filastò <art at fuffa.org>
Date:   Thu Aug 22 18:10:10 2013 +0200

    Implement downloading of input files based on URL in deck file
---
 ooni/deck.py        |   21 ++++++++++++++-------
 ooni/nettest.py     |    5 +++--
 ooni/oonibclient.py |    8 +++++---
 ooni/oonicli.py     |    2 +-
 ooni/utils/net.py   |    1 +
 5 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/ooni/deck.py b/ooni/deck.py
index ef1b58a..8f1335d 100644
--- a/ooni/deck.py
+++ b/ooni/deck.py
@@ -2,15 +2,19 @@
 
 from ooni.nettest import NetTestLoader
 from ooni.settings import config
+from ooni.utils import log
 from ooni.utils.txagentwithsocks import Agent
 from ooni.errors import UnableToLoadDeckInput
+from ooni.oonibclient import OONIBClient
+
 from twisted.internet import reactor, defer
+
 import os
 import re
 import yaml
 
 class Deck(object):
-    def __init__(self, oonibclient, deckFile=None):
+    def __init__(self, deckFile=None):
         self.netTestLoaders = []
         self.inputs = []
 
@@ -30,22 +34,25 @@ class Deck(object):
         self.fetchAndVerifyNetTestInput(net_test_loader)
         self.netTestLoaders.append(net_test_loader)
  
+    @defer.inlineCallbacks
     def fetchAndVerifyDeckInputs(self):
         """ fetch and verify inputs for all NetTests in the deck """
         for net_test_loader in self.netTestLoaders:
-            self.fetchAndVerifyNetTestInput(net_test_loader)
+            yield self.fetchAndVerifyNetTestInput(net_test_loader)
 
     @defer.inlineCallbacks
     def fetchAndVerifyNetTestInput(self, net_test_loader):
         """ fetch and verify a single NetTest's inputs """
-        for input_file in net_test_loader.inputFiles:
-            if 'url' in input_file:
-                oonib = OONIBClient(input_file['address'])
+        log.debug("Fetching and verifying inputs")
+        for i in net_test_loader.inputFiles:
+            if 'url' in i:
+                log.debug("Downloading %s" % i['url'])
+                oonib = OONIBClient(i['address'])
 
-                input_file = yield oonib.downloadInput(input_file['hash'])
+                input_file = yield oonib.downloadInput(i['hash'])
                 try:
                     input_file.verify()
                 except AssertionError:
                     raise UnableToLoadDeckInput, cached_path
                 
-                test_class.localOptions[input_file['key']] = input_file.cached_file
+                i['test_class'].localOptions[i['key']] = input_file.cached_file
diff --git a/ooni/nettest.py b/ooni/nettest.py
index 4806bdd..b8cdf65 100644
--- a/ooni/nettest.py
+++ b/ooni/nettest.py
@@ -173,7 +173,7 @@ class NetTestLoader(object):
     method_prefix = 'test'
 
     def __init__(self, options, test_file=None, test_string=None):
-        self.onionInputRegex =  re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]){40}$")
+        self.onionInputRegex =  re.compile("(httpo://[a-z0-9]{16}\.onion)/input/([a-z0-9]{64})$")
         self.options = options
         self.testCases, test_cases = None, None
 
@@ -196,7 +196,8 @@ class NetTestLoader(object):
                 key = test_class.inputFile[0]
                 filename = test_class.localOptions[key]
                 input_file = {
-                    'id': key
+                    'key': key,
+                    'test_class': test_class
                 }
                 m = self.onionInputRegex.match(filename)
                 if m:
diff --git a/ooni/oonibclient.py b/ooni/oonibclient.py
index 9665ee1..9d2085b 100644
--- a/ooni/oonibclient.py
+++ b/ooni/oonibclient.py
@@ -4,7 +4,8 @@ import json
 from hashlib import sha256
 
 from twisted.internet import defer, reactor
-from twisted.web.client import Agent
+
+from ooni.utils.txagentwithsocks import Agent
 
 from ooni.settings import config
 from ooni.utils import log
@@ -66,7 +67,8 @@ class InputFile(object):
 class OONIBClient(object):
     def __init__(self, address):
         self.address = address
-        self.agent = Agent(reactor)
+        self.agent = Agent(reactor, sockshost="127.0.0.1", 
+                           socksport=config.tor.socks_port)
         self.input_files = {}
 
     def _request(self, method, urn, genReceiver, bodyProducer=None):
@@ -77,7 +79,7 @@ class OONIBClient(object):
 
         @d.addCallback
         def callback(response):
-            content_length = response.headers.getRawHeaders('content-length')
+            content_length = int(response.headers.getRawHeaders('content-length')[0])
             response.deliverBody(genReceiver(finished, content_length))
 
         @d.addErrback
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 8708642..a7df18f 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -149,7 +149,7 @@ def runWithDirector():
 
     def fetch_nettest_inputs(result):
         try: 
-            deck.fetchAndVerifyDeckInputs()
+            return deck.fetchAndVerifyDeckInputs()
         except errors.UnableToLoadDeckInput, e:
             return defer.failure.Failure(result)
 
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index ab33d00..1ec9608 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -97,6 +97,7 @@ class Downloader(protocol.Protocol):
                 self.bytes_remaining -= len(b)
 
     def connectionLost(self, reason):
+        self.fp.flush()
         self.fp.close()
         self.finished.callback(None)