[tor-commits] [ooni-probe/master] Implement basic HTTP request test that does capitalization variations on the HTTP method
art at torproject.org
art at torproject.org
Sat Nov 10 17:21:20 UTC 2012
commit 2266e961595a167b12ee1a4aca617eb2dbb56b12
Author: Arturo Filastò <art at fuffa.org>
Date: Sat Nov 10 17:50:22 2012 +0100
Implement basic HTTP request test that does capitalization variations on the HTTP method
* Due to a quirk in twisted.client.Agent that sets HTTP request headers to canonical form we are
currently unable to test the request headers.
This is ticketized here: https://trac.torproject.org/projects/tor/ticket/7432
---
nettests/core/http_requests.py | 90 +++++++++++++++++++++++++++++++++-------
ooni/templates/httpt.py | 58 ++++++++++++++------------
oonib/testhelpers/httph.py | 26 ++++++++----
3 files changed, 124 insertions(+), 50 deletions(-)
diff --git a/nettests/core/http_requests.py b/nettests/core/http_requests.py
index 4fc8205..0315145 100644
--- a/nettests/core/http_requests.py
+++ b/nettests/core/http_requests.py
@@ -4,6 +4,9 @@
# :licence: see LICENSE
import random
+import json
+
+from ooni.utils import log, net
from ooni.templates import httpt
def random_capitalization(string):
@@ -21,11 +24,16 @@ class HTTPRequests(httpt.HTTPTest):
This test is also known as Header Field manipulation. It performes HTTP
requests with variations in capitalization towards the backend.
"""
- name = "HTTPRequests"
+ name = "HTTP Requests"
author = "Arturo Filastò"
version = 0.1
- optParameters = [['backend', 'b', None, 'URL of the backend to use for sending the requests']]
+ optParameters = [
+ ['backend', 'b', None,
+ 'URL of the backend to use for sending the requests'],
+ ['headers', 'h', None,
+ 'Specify a yaml formatted file from which to read the request headers to send']
+ ]
requiredOptions = ['backend']
@@ -35,24 +43,76 @@ class HTTPRequests(httpt.HTTPTest):
else:
raise Exception("No backend specified")
+ def processResponseBody(self, data):
+ response = json.loads(data)
+ if response['request_method'] != self.request_method:
+ self.report['tampering'] = True
+ else:
+ self.report['tampering'] = False
+ # XXX add checks for validation of sent headers
+
+ def get_headers(self):
+ headers = {}
+ if self.localOptions['headers']:
+ # XXX test this code
+ try:
+ f = open(self.localOptions['headers'])
+ except IOError:
+ raise Exception("Specified input file does not exist")
+ content = ''.join(f.readlines())
+ f.close()
+ headers = yaml.load(content)
+ return headers
+ else:
+ headers = {"User-Agent": [random.choice(net.userAgents)],
+ "Accept": ["text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],
+ "Accept-Encoding": ["gzip,deflate,sdch"],
+ "Accept-Language": ["en-US,en;q=0.8"],
+ "Accept-Charset": ["ISO-8859-1,utf-8;q=0.7,*;q=0.3"]}
+ return headers
+
+ def get_random_caps_headers(self):
+ headers = {}
+ normal_headers = self.get_headers()
+ for k, v in normal_headers.items():
+ new_key = random_capitalization(k)
+ headers[new_key] = v
+ return headers
+
def test_get(self):
- return self.doRequest(self.url, "GET")
+ self.request_method = "GET"
+ self.request_headers = self.get_random_caps_headers()
+ return self.doRequest(self.url, self.request_method,
+ headers=self.request_headers)
- def test_get_random_capitalization(self):
- method = random_capitalization("GET")
- return self.doRequest(self.url, method)
+ def a_test_get_random_capitalization(self):
+ self.request_method = random_capitalization("GET")
+ self.request_headers = self.get_random_caps_headers()
+ return self.doRequest(self.url, self.request_method,
+ headers=self.request_headers)
- def test_post(self):
- return self.doRequest(self.url, "POST")
+ def a_test_post(self):
+ self.request_method = "POST"
+ self.request_headers = self.get_headers()
+ return self.doRequest(self.url, self.request_method,
+ headers=self.request_headers)
- def test_post_random_capitalization(self):
- method = random_capitalization("POST")
- return self.doRequest(self.url, method)
+ def a_test_post_random_capitalization(self):
+ self.request_method = random_capitalization("POST")
+ self.request_headers = self.get_random_caps_headers()
+ return self.doRequest(self.url, self.request_method,
+ headers=self.request_headers)
- def test_put(self):
- return self.doRequest(self.url, "PUT")
+ def a_test_put(self):
+ self.request_method = "PUT"
+ self.request_headers = self.get_headers()
+ return self.doRequest(self.url, self.request_method,
+ headers=self.request_headers)
def test_put_random_capitalization(self):
- method = random_capitalization("PUT")
- return self.doRequest(self.url, method)
+ self.request_method = random_capitalization("PUT")
+ self.request_headers = self.get_random_caps_headers()
+ return self.doRequest(self.url, self.request_method,
+ headers=self.request_headers)
+
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 4c42a3a..96b7cc8 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -2,7 +2,7 @@
#
# :authors: Arturo Filastò
# :licence: see LICENSE
-
+import copy
import random
from zope.interface import implements
@@ -12,6 +12,10 @@ from twisted.plugin import IPlugin
from twisted.internet import protocol, defer
from twisted.internet.ssl import ClientContextFactory
+from twisted.web.client import Agent
+from twisted.internet import reactor
+
+from twisted.web._newclient import Request
from twisted.web.http_headers import Headers
from ooni.nettest import NetTestCase
from ooni.utils import log
@@ -39,8 +43,6 @@ class HTTPTest(NetTestCase):
except:
log.err("Warning! pyOpenSSL is not installed. https websites will"
"not work")
- from twisted.web.client import Agent
- from twisted.internet import reactor
self.agent = Agent(reactor)
@@ -107,7 +109,7 @@ class HTTPTest(NetTestCase):
method: the HTTP Method to be used
- headers: the request headers to be sent
+ headers: the request headers to be sent as a dict
body: the request body
@@ -137,6 +139,31 @@ class HTTPTest(NetTestCase):
d.addCallback(finished)
return d
+ def build_request(self, url, method="GET", headers=None, body=None):
+ self.request['method'] = method
+ self.request['url'] = url
+ self.request['headers'] = headers if headers else {}
+ self.request['body'] = body
+
+ if self.randomizeUA:
+ self.randomize_useragent()
+
+ self.report['request'] = self.request
+ self.report['url'] = url
+
+ # If we have a request body payload, set the request body to such
+ # content
+ if body:
+ body_producer = StringProducer(self.request['body'])
+ else:
+ body_producer = None
+
+ headers = Headers(self.request['headers'])
+
+ req = self.agent.request(self.request['method'], self.request['url'],
+ headers, body_producer)
+ return req
+
def _cbResponse(self, response, headers_processor, body_processor):
log.debug("Got response %s" % response)
if not response:
@@ -167,27 +194,4 @@ class HTTPTest(NetTestCase):
user_agent = random.choice(userAgents)
self.request['headers']['User-Agent'] = [user_agent]
- def build_request(self, url, method="GET", headers=None, body=None):
- self.request['method'] = method
- self.request['url'] = url
- self.request['headers'] = headers if headers else {}
- self.request['body'] = body
-
- if self.randomizeUA:
- self.randomize_useragent()
-
- self.report['request'] = self.request
- self.report['url'] = url
-
- # If we have a request body payload, set the request body to such
- # content
- if body:
- body_producer = StringProducer(self.request['body'])
- else:
- body_producer = None
-
- req = self.agent.request(self.request['method'], self.request['url'],
- Headers(self.request['headers']),
- body_producer)
- return req
diff --git a/oonib/testhelpers/httph.py b/oonib/testhelpers/httph.py
index b793852..7001fa6 100644
--- a/oonib/testhelpers/httph.py
+++ b/oonib/testhelpers/httph.py
@@ -12,16 +12,26 @@ from cyclone.web import RequestHandler, Application
class HTTPTrapAll(RequestHandler):
"""
- Master class to be used to trap all the HTTP methods
+ Master class to be used to trap all the HTTP methods and make capitalized
+ requests pass.
"""
- def get(self, *arg, **kw):
- self.all(*arg, **kw)
+ def _execute(self, transforms, *args, **kwargs):
+ self._transforms = transforms
+ defer.maybeDeferred(self.prepare).addCallbacks(
+ self._execute_handler,
+ lambda f: self._handle_request_exception(f.value),
+ callbackArgs=(args, kwargs))
- def post(self, *arg, **kw):
- self.all(*arg, **kw)
-
- def put(self, *arg, **kw):
- self.all(*arg, **kw)
+ def _execute_handler(self, r, args, kwargs):
+ if not self._finished:
+ args = [self.decode_argument(arg) for arg in args]
+ kwargs = dict((k, self.decode_argument(v, name=k))
+ for (k, v) in kwargs.iteritems())
+ # This is where we do the patching
+ # XXX this is somewhat hackish
+ d = defer.maybeDeferred(self.all, *args, **kwargs)
+ d.addCallbacks(self._execute_success, self._execute_failure)
+ self.notifyFinish().addCallback(self.on_connection_close)
class HTTPReturnJSONHeaders(HTTPTrapAll):
def all(self):
More information about the tor-commits
mailing list