[tor-commits] [stem/master] Download method for microdescriptors

[atagar at torproject.org]

commit 60d714a33ca055fcefdad26493b4787483b1d71a
Author: Damian Johnson <atagar at torproject.org>
Date:   Wed Aug 7 18:36:53 2019 -0700

    Download method for microdescriptors
---
 stem/descriptor/collector.py       | 39 +++++++++++++++++++++++++++++++++++++-
 test/integ/descriptor/collector.py | 13 ++++++++++++-
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index 82a213f3..c8c4a1b7 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -50,7 +50,8 @@ With this you can either download and read directly from CollecTor...
 
   get_instance - Provides a singleton CollecTor used for...
     |- get_server_descriptors - published server descriptors
-    +- get_extrainfo_descriptors - published extrainfo descriptors
+    |- get_extrainfo_descriptors - published extrainfo descriptors
+    +- get_microdescriptors - published microdescriptors
 
   File - Individual file residing within CollecTor
     |- read - provides descriptors from this file
@@ -59,6 +60,7 @@ With this you can either download and read directly from CollecTor...
   CollecTor - Downloader for descriptors from CollecTor
     |- get_server_descriptors - published server descriptors
     |- get_extrainfo_descriptors - published extrainfo descriptors
+    |- get_microdescriptors - published microdescriptors
     |
     |- index - metadata for content available from CollecTor
     +- files - files available from CollecTor
@@ -163,6 +165,17 @@ def get_extrainfo_descriptors(start = None, end = None, cache_to = None, timeout
     yield desc
 
 
+def get_microdescriptors(start = None, end = None, cache_to = None, timeout = None, retries = 3):
+  """
+  Shorthand for
+  :func:`~stem.descriptor.collector.CollecTor.get_microdescriptors`
+  on our singleton instance.
+  """
+
+  for desc in get_instance().get_microdescriptors(start, end, cache_to, timeout, retries):
+    yield desc
+
+
 class File(object):
   """
   File within CollecTor.
@@ -430,6 +443,30 @@ class CollecTor(object):
       for desc in f.read(cache_to, timeout = timeout, retries = retries):
         yield desc
 
+  def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
+    """
+    Provides microdescriptors published during the given time range,
+    sorted oldest to newest.
+
+    :param datetime.datetime start: time range to begin with
+    :param datetime.datetime end: time range to end with
+    :param str cache_to: directory to cache archives into, if an archive is
+      available here it is not downloaded
+    :param int timeout: timeout for downloading each individual archive when
+      the connection becomes idle, no timeout applied if **None**
+    :param int retires: maximum attempts to impose on a per-archive basis
+
+    :returns: **iterator** of
+      :class:`~stem.descriptor.microdescriptor.Microdescriptor
+      for the given time range
+
+    :raises: :class:`~stem.DownloadFailed` if the download fails
+    """
+
+    for f in self.files('microdescriptor', start, end):
+      for desc in f.read(cache_to, timeout = timeout, retries = retries):
+        yield desc
+
   def index(self, compression = 'best'):
     """
     Provides the archives available in CollecTor.
diff --git a/test/integ/descriptor/collector.py b/test/integ/descriptor/collector.py
index 1a2e1e32..db57df5e 100644
--- a/test/integ/descriptor/collector.py
+++ b/test/integ/descriptor/collector.py
@@ -49,7 +49,18 @@ class TestCollector(unittest.TestCase):
     recent_descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors(start = RECENT))
 
     if not (300 < len(recent_descriptors) < 800):
-      self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors))
+      self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors))  # 583 on 8/7/19
+
+  @test.require.only_run_once
+  @test.require.online
+  def test_downloading_microdescriptors(self):
+    recent_descriptors = list(stem.descriptor.collector.get_microdescriptors(start = RECENT))
+
+    # TODO: I'm unsure why these counts differ so much from server/extrainfo
+    # descriptors. Checking with Karsten.
+
+    if not (300 < len(recent_descriptors) < 800):
+      self.fail('Downloaded %i descriptors, expected 300-800' % len(recent_descriptors))  # 23 on 8/7/19
 
   def _test_index(self, compression):
     if compression and not compression.available: