[tor-commits] [stem/master] Download method for TorDNSEL exit lists

[atagar at torproject.org]

commit 466f319f3bb0a40317bd30dfd3fee487d7bb3432
Author: Damian Johnson <atagar at torproject.org>
Date:   Thu Aug 15 14:29:04 2019 -0700

    Download method for TorDNSEL exit lists
---
 stem/descriptor/collector.py                       |  39 ++++++++++++++++++++-
 test/unit/descriptor/collector.py                  |  15 ++++++++
 .../data/collector/exit-list-2018-11-cropped.tar   | Bin 0 -> 590336 bytes
 3 files changed, 53 insertions(+), 1 deletion(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 9a238ecf..6a4c700f 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -53,7 +53,8 @@ With this you can either download and read directly from CollecTor...
     |- get_extrainfo_descriptors - published extrainfo descriptors
     |- get_microdescriptors - published microdescriptors
     |- get_consensus - published router status entries
-    +- get_key_certificates - authority key certificates
+    |- get_key_certificates - authority key certificates
+    +- get_exit_list - TorDNSEL exit list
 
   File - Individual file residing within CollecTor
     |- read - provides descriptors from this file
@@ -65,6 +66,7 @@ With this you can either download and read directly from CollecTor...
     |- get_microdescriptors - published microdescriptors
     |- get_consensus - published router status entries
     |- get_key_certificates - authority key certificates
+    |- get_exit_list - TorDNSEL exit list
     |
     |- index - metadata for content available from CollecTor
     +- files - files available from CollecTor
@@ -203,6 +205,17 @@ def get_key_certificates(start = None, end = None, cache_to = None, timeout = No
     yield desc
 
 
+def get_exit_list(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+  """
+  Shorthand for
+  :func:`~stem.descriptor.collector.CollecTor.get_exit_list`
+  on our singleton instance.
+  """
+
+  for desc in get_instance().get_exit_list(start, end, cache_to, timeout, retries):
+    yield desc
+
+
 class File(object):
   """
   File within CollecTor.
@@ -581,6 +594,30 @@ class CollecTor(object):
       for desc in f.read(cache_to, 'dir-key-certificate-3', timeout = timeout, retries = retries):
         yield desc
 
+  def get_exit_list(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+    """
+    `TorDNSEL exit list <https://www.torproject.org/projects/tordnsel.html.en>`_
+    for the given time range, sorted oldest to newest.
+
+    :param datetime.datetime start: time range to begin with
+    :param datetime.datetime end: time range to end with
+    :param str cache_to: directory to cache archives into, if an archive is
+      available here it is not downloaded
+    :param int timeout: timeout for downloading each individual archive when
+      the connection becomes idle, no timeout applied if **None**
+    :param int retries: maximum attempts to impose on a per-archive basis
+
+    :returns: **iterator** of
+      :class:`~stem.descriptor.tordnsel.TorDNSEL
+      for the given time range
+
+    :raises: :class:`~stem.DownloadFailed` if the download fails
+    """
+
+    for f in self.files('tordnsel', start, end):
+      for desc in f.read(cache_to, 'tordnsel', timeout = timeout, retries = retries):
+        yield desc
+
   def index(self, compression = 'best'):
     """
     Provides the archives available in CollecTor.
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index d6a3aee8..e3ba1f40 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -341,3 +341,18 @@ class TestCollector(unittest.TestCase):
     f = descriptors[0]
     self.assertEqual('KeyCertificate', type(f).__name__)
     self.assertEqual('14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4', f.fingerprint)
+
+  @patch('stem.util.connection.download')
+  @patch('stem.descriptor.collector.CollecTor.files')
+  def test_reading_exit_list(self, files_mock, download_mock):
+    with open(get_resource('collector/exit-list-2018-11-cropped.tar'), 'rb') as archive:
+      download_mock.return_value = archive.read()
+
+    files_mock.return_value = [stem.descriptor.collector.File('archive/exit-lists/exit-list-2018-11.tar', 12345, '2016-09-04 09:21')]
+
+    descriptors = list(stem.descriptor.collector.get_exit_list())
+    self.assertEqual(3713, len(descriptors))
+
+    f = descriptors[0]
+    self.assertEqual('TorDNSEL', type(f).__name__)
+    self.assertEqual('0011BD2485AD45D984EC4159C88FC066E5E3300E', f.fingerprint)
diff --git a/test/unit/descriptor/data/collector/exit-list-2018-11-cropped.tar b/test/unit/descriptor/data/collector/exit-list-2018-11-cropped.tar
new file mode 100644
index 00000000..2f86f8de
Binary files /dev/null and b/test/unit/descriptor/data/collector/exit-list-2018-11-cropped.tar differ