[tor-commits] [ooni-probe/master] Also record all the redirects

[art at torproject.org]

commit 8a745093ecd27240053b7e251de46eb4c303df6f
Author: Arturo Filastò <arturo at filasto.net>
Date:   Mon Apr 18 18:45:29 2016 +0200

    Also record all the redirects
    
    * Add monkey patch for bug in twisted RedirectAgent: https://twistedmatrix.com/trac/ticket/8265
---
 ooni/templates/httpt.py   | 22 ++++++++++++++--------
 ooni/utils/trueheaders.py | 42 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 2b280f0..2a17f5b 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -7,6 +7,7 @@ from twisted.web.client import readBody, PartialDownloadError
 from twisted.internet import reactor
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent
+from ooni.utils.trueheaders import FixedRedirectAgent
 
 from ooni.nettest import NetTestCase
 from ooni.utils import log, base64Dict
@@ -37,8 +38,15 @@ class StreamListener(StreamListenerMixin):
             log.err("Tor Exit ip detection failed")
 
 
+def _representHeaders(headers):
+    represented_headers = {}
+    for name, value in headers.getAllRawHeaders():
+        represented_headers[name] = value[0]
+    return represented_headers
 
 def _representBody(body):
+    if not body:
+        return body
     # XXX perhaps add support for decoding gzip in the future.
     body = body.replace('\0', '')
     decoded = False
@@ -115,9 +123,8 @@ class HTTPTest(NetTestCase):
 
         if self.followRedirects:
             try:
-                from twisted.web.client import RedirectAgent
-                self.control_agent = RedirectAgent(self.control_agent)
-                self.agent = RedirectAgent(self.agent)
+                self.control_agent = FixedRedirectAgent(self.control_agent)
+                self.agent = FixedRedirectAgent(self.agent)
                 self.report['agent'] = 'redirect'
             except:
                 log.err("Warning! You are running an old version of twisted"\
@@ -147,11 +154,10 @@ class HTTPTest(NetTestCase):
 
             failure (instance): An instance of :class:twisted.internet.failure.Failure
         """
-        def _representHeaders(headers):
-            represented_headers = {}
-            for name, value in headers.getAllRawHeaders():
-                represented_headers[name] = value[0]
-            return represented_headers
+        if response and response.previousResponse:
+            self.addToReport(request, response.previousResponse,
+                             response_body=None,
+                             failure_string=None)
 
         log.debug("Adding %s to report" % request)
         request_headers = TrueHeaders(request['headers'])
diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py
index d5aa206..a5b2fe3 100644
--- a/ooni/utils/trueheaders.py
+++ b/ooni/utils/trueheaders.py
@@ -7,9 +7,14 @@
 import itertools
 from copy import copy
 
-from twisted.web import client, _newclient, http_headers
+from twisted.python.failure import Failure
+
+from twisted.web import client, _newclient, http_headers, error
+
 from twisted.web._newclient import RequestNotSent, RequestGenerationFailed
 from twisted.web._newclient import TransportProxyProducer, STATUS
+from twisted.web._newclient import ResponseFailed
+
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, fail, maybeDeferred, failure
 
@@ -179,3 +184,38 @@ class TrueHeadersSOCKS5Agent(SOCKS5Agent):
             self._wrappedAgent._pool = pool
         else:
             self._pool = pool
+
+class FixedRedirectAgent(client.BrowserLikeRedirectAgent):
+    """
+    This is a redirect agent with this patch manually applied:
+    https://twistedmatrix.com/trac/ticket/8265
+    """
+    def _handleRedirect(self, response, method, uri, headers, redirectCount):
+        """
+        Handle a redirect response, checking the number of redirects already
+        followed, and extracting the location header fields.
+
+        This is pathed to fix a bug in infinite redirect loop.
+        """
+        if redirectCount >= self._redirectLimit:
+            err = error.InfiniteRedirection(
+                response.code,
+                b'Infinite redirection detected',
+                location=uri)
+            raise ResponseFailed([Failure(err)], response)
+        locationHeaders = response.headers.getRawHeaders(b'location', [])
+        if not locationHeaders:
+            err = error.RedirectWithNoLocation(
+                response.code, b'No location header field', uri)
+            raise ResponseFailed([Failure(err)], response)
+        location = self._resolveLocation(response.request.absoluteURI, locationHeaders[0])
+        deferred = self._agent.request(method, location, headers)
+
+        def _chainResponse(newResponse):
+            newResponse.setPreviousResponse(response)
+            return newResponse
+
+        deferred.addCallback(_chainResponse)
+        # This is the fix to properly handle redirects
+        return deferred.addCallback(
+            self._handleResponse, method, uri, headers, redirectCount + 1)