[tor-commits] [ooni-probe/master] Fully switch to using only cyclone for HTTP related test helpers
art at torproject.org
art at torproject.org
Sat Nov 10 14:02:09 UTC 2012
commit 7b3d5e71c710282d70f32fd81a6955e192119b17
Author: Arturo Filastò <art at fuffa.org>
Date: Sat Nov 10 14:16:02 2012 +0100
Fully switch to using only cyclone for HTTP related test helpers
* Implement HTTP Request handler backend
* Fix the startup script of OONIB
* Disable daphn3 test helper script
* Various cleanups
---
bin/oonib | 41 ++++++++++++++++--
ooni/utils/log.py | 18 +++++++-
oonib/lib/__init__.py | 7 ---
oonib/lib/ssl.py | 2 +-
oonib/oonibackend.py | 29 +++++--------
oonib/report/db/__init__.py | 4 +-
oonib/testhelpers/daphn3.py | 4 +-
oonib/testhelpers/httph.py | 100 +++++++++++++++++--------------------------
8 files changed, 107 insertions(+), 98 deletions(-)
diff --git a/bin/oonib b/bin/oonib
index df7c62f..0808743 100755
--- a/bin/oonib
+++ b/bin/oonib
@@ -1,4 +1,37 @@
-#!/bin/sh
-ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )"
-export PYTHONPATH=$PYTHONPATH:$ROOT
-twistd -ny $ROOT/oonib/oonibackend.py
+#!/usr/bin/env python
+# -*- encoding: utf-8 -*-
+#
+# :authors: Arturo Filastò
+# :licence: see LICENSE
+
+import sys
+import os
+# Avoid the export PYTHONPATH insanity
+sys.path[:] = map(os.path.abspath, sys.path)
+this_directory = os.path.dirname(__file__)
+root = os.path.abspath(os.path.join(this_directory, '..'))
+sys.path.insert(0, root)
+
+backend_script = os.path.join(root, 'oonib', 'oonibackend.py')
+
+from twisted.python import log, usage
+from twisted.internet import reactor
+from twisted.application import app
+
+from oonib import runner
+from oonib.oonibackend import application
+
+sys.argv[1:] = ['-ny', backend_script]
+
+def runApp(config):
+ runner.OBaseRunner(config).run()
+
+config = runner.ServerOptions()
+try:
+ config.parseOptions()
+except usage.error, ue:
+ print config
+ print "%s: %s" % (sys.argv[0], ue)
+else:
+ runApp(config)
+
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index 0cb62ac..c05c776 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -13,7 +13,7 @@ from twisted.python.logfile import DailyLogFile
from ooni.utils import otime
from ooni import config
-def start(logfile=None):
+def start(logfile=None, application_name="OONI"):
daily_logfile = None
if not logfile:
@@ -24,7 +24,7 @@ def start(logfile=None):
daily_logfile = DailyLogFile(log_filename, log_folder)
- txlog.msg("Starting OONI on %s (%s UTC)" % (otime.prettyDateNow(),
+ txlog.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(),
otime.utcPrettyDateNow()))
logging.basicConfig()
python_logging = txlog.PythonLoggingObserver()
@@ -53,3 +53,17 @@ def err(msg, *arg, **kw):
def exception(*msg):
logging.exception(msg)
+class LoggerFactory(object):
+ """
+ This is a logger factory to be used by oonib
+ """
+ def __init__(self, options):
+ pass
+
+ def start(self, application):
+ # XXX parametrize this
+ start('/tmp/oonib.log', "OONIB")
+
+ def stop(self):
+ txlog.msg("Stopping OONIB")
+
diff --git a/oonib/lib/__init__.py b/oonib/lib/__init__.py
index 5b4380a..8b13789 100644
--- a/oonib/lib/__init__.py
+++ b/oonib/lib/__init__.py
@@ -1,8 +1 @@
-from ooni.utils import Storage
-from ooni.utils.config import Config
-import os
-root = os.path.normpath(os.path.join(os.path.realpath(__file__), '../../'))
-config = Storage()
-config.main = Config('main', os.path.join(root, 'oonibackend.conf'))
-config.daphn3 = Config('daphn3', os.path.join(root, 'oonibackend.conf'))
diff --git a/oonib/lib/ssl.py b/oonib/lib/ssl.py
index 094bdc9..20e99e8 100644
--- a/oonib/lib/ssl.py
+++ b/oonib/lib/ssl.py
@@ -1,5 +1,5 @@
from twisted.internet import ssl
-from oonib.lib import config
+from oonib import config
class SSLContext(ssl.DefaultOpenSSLContextFactory):
def __init__(self, *args, **kw):
diff --git a/oonib/oonibackend.py b/oonib/oonibackend.py
index eaa52f9..9964854 100644
--- a/oonib/oonibackend.py
+++ b/oonib/oonibackend.py
@@ -24,36 +24,26 @@ from ooni.utils import log
from oonib.report.api import reportingBackend
from oonib.lib.ssl import SSLContext
-from oonib.lib import config
+from oonib import config
-from oonib.testhelpers.httph import HTTPBackend, DebugHTTPServer
+from oonib.testhelpers.httph import HTTPReturnJSONHeadersHelper
from oonib.testhelpers.dns import ProxyDNSServer
-from oonib.testhelpers.daphn3 import Daphn3Server
+#from oonib.testhelpers.daphn3 import Daphn3Server
from cyclone import web
-log.start('/tmp/oonib.log')
-
# This tells twisted to set the
server.version = config.main.server_version
application = service.Application('oonibackend')
serviceCollection = service.IServiceCollection(application)
-if config.main.http_port:
- internet.TCPServer(int(config.main.http_port),
- server.Site(HTTPBackend())
- ).setServiceParent(serviceCollection)
-
if config.main.ssl_port:
internet.SSLServer(int(config.main.ssl_port),
server.Site(HTTPBackend()),
SSLContext(config),
).setServiceParent(serviceCollection)
-debugHTTPServer = DebugHTTPServer()
-internet.TCPServer(8090, debugHTTPServer).setServiceParent(serviceCollection)
-
# Start the DNS Server related services
if config.main.dns_tcp_port:
TCPDNSServer = ProxyDNSServer()
@@ -65,15 +55,18 @@ if config.main.dns_udp_port:
internet.UDPServer(int(config.main.dns_udp_port),
UDPFactory).setServiceParent(serviceCollection)
+# XXX this needs to be ported
# Start the OONI daphn3 backend
-if config.main.daphn3_port:
- daphn3 = Daphn3Server()
- internet.TCPServer(int(config.main.daphn3_port),
- daphn3).setServiceParent(serviceCollection)
+#if config.main.daphn3_port:
+# daphn3 = Daphn3Server()
+# internet.TCPServer(int(config.main.daphn3_port),
+# daphn3).setServiceParent(serviceCollection)
if config.main.reporting_port:
internet.TCPServer(int(config.main.reporting_port),
reportingBackend).setServiceParent(serviceCollection)
-reactor.addSystemEventTrigger('after', 'shutdown', db_threadpool.stop)
+if config.helpers.http_return_request_port:
+ internet.TCPServer(int(config.helpers.http_return_request_port),
+ HTTPReturnJSONHeadersHelper).setServiceParent(serviceCollection)
diff --git a/oonib/report/db/__init__.py b/oonib/report/db/__init__.py
index 178abe9..383b24b 100644
--- a/oonib/report/db/__init__.py
+++ b/oonib/report/db/__init__.py
@@ -1,5 +1,3 @@
-__all__ = ['models']
-
from twisted.python import log
from twisted.python.threadpool import ThreadPool
from twisted.internet.defer import inlineCallbacks
@@ -21,7 +19,7 @@ def getStore():
@inlineCallbacks
def create_tables():
- from oonib.report.db import models
+ from oonib.report.models import models
for x in models.__all__:
query = getattr(models.__getattribute__(x), 'createQuery')
diff --git a/oonib/testhelpers/daphn3.py b/oonib/testhelpers/daphn3.py
index facd11f..3b59f93 100644
--- a/oonib/testhelpers/daphn3.py
+++ b/oonib/testhelpers/daphn3.py
@@ -1,9 +1,9 @@
+# XXX this is currently broken and needs to be ported
from twisted.internet import protocol
from twisted.internet.error import ConnectionDone
-from oonib.lib import config
+from oonib import config
-from ooni.plugoo import reports
from ooni.protocols.daphn3 import Mutator, Daphn3Protocol
from ooni.protocols.daphn3 import read_pcap, read_yaml
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
index 217dbd5..b793852 100644
--- a/oonib/testhelpers/httph.py
+++ b/oonib/testhelpers/httph.py
@@ -8,14 +8,42 @@ from twisted.protocols import basic
from twisted.web import resource, server, static, http
from twisted.web.microdom import escape
-server.version = "Apache"
+from cyclone.web import RequestHandler, Application
-class HTTPRandomPage(resource.Resource):
+class HTTPTrapAll(RequestHandler):
+ """
+ Master class to be used to trap all the HTTP methods
+ """
+ def get(self, *arg, **kw):
+ self.all(*arg, **kw)
+
+ def post(self, *arg, **kw):
+ self.all(*arg, **kw)
+
+ def put(self, *arg, **kw):
+ self.all(*arg, **kw)
+
+class HTTPReturnJSONHeaders(HTTPTrapAll):
+ def all(self):
+ # XXX make sure that the request headers are in the correct order
+ submitted_data = {'request_body': self.request.body,
+ 'request_headers': self.request.headers,
+ 'request_uri': self.request.uri,
+ 'request_method': self.request.method}
+ response = json.dumps(submitted_data)
+ self.write(response)
+
+HTTPReturnJSONHeadersHelper = Application([
+ (r"/*", HTTPReturnJSONHeaders)
+])
+
+
+class HTTPRandomPage(HTTPTrapAll):
"""
This generates a random page of arbitrary length and containing the string
selected by the user.
- The format is the following:
- /random/<length>/<keyword>
+ /<length>/<keyword>
+ XXX this is currently disabled as it is not of use to any test.
"""
isLeaf = True
def _gen_random_string(self, length):
@@ -29,64 +57,14 @@ class HTTPRandomPage(resource.Resource):
data += '\n'
return data
- def render(self, request):
+ def all(self, length, keyword):
length = 100
- keyword = None
- path_parts = request.path.split('/')
- if len(path_parts) > 2:
- length = int(path_parts[2])
- if length > 100000:
- length = 100000
-
- if len(path_parts) > 3:
- keyword = escape(path_parts[3])
-
+ if length > 100000:
+ length = 100000
return self.genRandomPage(length, keyword)
-class HTTPReturnHeaders(resource.Resource):
- """
- This returns the headers being sent by the client in JSON format.
- """
- isLeaf = True
- def render(self, request):
- req_headers = request.getAllHeaders()
- return json.dumps(req_headers)
-
-class HTTPSendHeaders(resource.Resource):
- """
- This sends to the client the headers that they send inside of the POST
- request encoded in json.
- """
- isLeaf = True
- def render_POST(self, request):
- headers = json.loads(request.content.read())
- for header, value in headers.items():
- request.setHeader(str(header), str(value))
- return ''
-
-class HTTPBackend(resource.Resource):
- def __init__(self):
- resource.Resource.__init__(self)
- self.putChild('random', HTTPRandomPage())
- self.putChild('returnheaders', HTTPReturnHeaders())
- self.putChild('sendheaders', HTTPSendHeaders())
-
-class DebugProtocol(http.HTTPChannel):
- def headerReceived(self, line):
- print "[HEADER] %s" % line
- http.HTTPChannel.headerReceived(self, line)
-
- def allContentReceived(self):
- print self.requests[-1].getAllHeaders()
- self.transport.loseConnection()
- self.connectionLost("Normal closure")
-
-class DebugHTTPServer(http.HTTPFactory):
- protocol = DebugProtocol
-
- def buildProtocol(self, addr):
- print "Got connection from %s" % addr
- p = protocol.ServerFactory.buildProtocol(self, addr)
- p.timeOut = self.timeOut
- return p
+HTTPRandomPageHelper = Application([
+ # XXX add regexps here
+ (r"/(.*)/(.*)", HTTPRandomPage)
+])
More information about the tor-commits
mailing list