[tor-commits] [stem/master] Unit test collector downloading
atagar at torproject.org
atagar at torproject.org
Sat Aug 17 20:44:27 UTC 2019
commit 4a319f1794e54fc216d84ac192cc3efc1ce15ec2
Author: Damian Johnson <atagar at torproject.org>
Date: Sun Aug 11 05:14:49 2019 -0700
Unit test collector downloading
On reflection there's no need for us to rely exclusively on integ tests for
download coverage. Those certainly do a great job, but simple to get unit test
coverage of this as well.
---
stem/descriptor/collector.py | 8 +-
test/unit/descriptor/collector.py | 81 ++++++++++++++++++++-
test/unit/descriptor/data/__init__.py | 2 +-
test/unit/descriptor/data/collector/__init__.py | 7 ++
.../data/collector/consensuses-2018-06-cropped.tar | Bin 0 -> 100864 bytes
.../data/collector/extra-infos-2019-04-cropped.tar | Bin 0 -> 22528 bytes
.../{collector_index.json => collector/index.json} | 0
.../{collector_index.py => collector/index.py} | 0
.../data/collector/microdescs-2019-05-cropped.tar | Bin 0 -> 193024 bytes
.../server-descriptors-2005-12-cropped.tar | Bin 0 -> 22528 bytes
10 files changed, 91 insertions(+), 7 deletions(-)
diff --git a/stem/descriptor/collector.py b/stem/descriptor/collector.py
index c0c97832..09e94e74 100644
--- a/stem/descriptor/collector.py
+++ b/stem/descriptor/collector.py
@@ -435,7 +435,7 @@ class CollecTor(object):
# TODO: support bridge variants ('bridge-server-descriptor' type)
for f in self.files('server-descriptor', start, end):
- for desc in f.read(cache_to, timeout = timeout, retries = retries):
+ for desc in f.read(cache_to, 'server-descriptor', timeout = timeout, retries = retries):
yield desc
def get_extrainfo_descriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
@@ -461,7 +461,7 @@ class CollecTor(object):
# TODO: support bridge variants ('bridge-extra-info' type)
for f in self.files('extra-info', start, end):
- for desc in f.read(cache_to, timeout = timeout, retries = retries):
+ for desc in f.read(cache_to, 'extra-info', timeout = timeout, retries = retries):
yield desc
def get_microdescriptors(self, start = None, end = None, cache_to = None, timeout = None, retries = 3):
@@ -494,7 +494,7 @@ class CollecTor(object):
"""
for f in self.files('microdescriptor', start, end):
- for desc in f.read(cache_to, timeout = timeout, retries = retries):
+ for desc in f.read(cache_to, 'microdescriptor', timeout = timeout, retries = retries):
yield desc
def get_consensus(self, start = None, end = None, cache_to = None, version = 3, microdescriptor = False, timeout = None, retries = 3):
@@ -536,7 +536,7 @@ class CollecTor(object):
# TODO: document vs router status entries (ie. DocumentType)?
for f in self.files(desc_type, start, end):
- for desc in f.read(cache_to, timeout = timeout, retries = retries):
+ for desc in f.read(cache_to, desc_type, timeout = timeout, retries = retries):
yield desc
def index(self, compression = 'best'):
diff --git a/test/unit/descriptor/collector.py b/test/unit/descriptor/collector.py
index 77c1c460..f5bed87e 100644
--- a/test/unit/descriptor/collector.py
+++ b/test/unit/descriptor/collector.py
@@ -11,7 +11,7 @@ import stem.prereq
from stem.descriptor import Compression
from stem.descriptor.collector import CollecTor, File
from test.unit.descriptor import get_resource
-from test.unit.descriptor.data.collector_index import EXAMPLE_INDEX
+from test.unit.descriptor.data.collector.index import EXAMPLE_INDEX
try:
# added in python 3.3
@@ -22,7 +22,7 @@ except ImportError:
URL_OPEN = 'urllib.request.urlopen' if stem.prereq.is_python_3() else 'urllib2.urlopen'
-with open(get_resource('collector_index.json'), 'rb') as index_file:
+with open(get_resource('collector/index.json'), 'rb') as index_file:
EXAMPLE_INDEX_JSON = index_file.read()
@@ -194,3 +194,80 @@ class TestCollector(unittest.TestCase):
self.assertEqual([
'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz',
], [f.path for f in collector.files(descriptor_type = 'server-descriptor', start = datetime.datetime(2006, 2, 10), end = datetime.datetime(2007, 1, 1))])
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_server_descriptors(self, files_mock, download_mock):
+ with open(get_resource('collector/server-descriptors-2005-12-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_server_descriptors())
+ self.assertEqual(5, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('RelayDescriptor', type(f).__name__)
+ self.assertEqual('3E2F63E2356F52318B536A12B6445373808A5D6C', f.fingerprint)
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_extrainfo_descriptors(self, files_mock, download_mock):
+ with open(get_resource('collector/extra-infos-2019-04-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/extra-infos/extra-infos-2019-04.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_extrainfo_descriptors())
+ self.assertEqual(7, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('RelayExtraInfoDescriptor', type(f).__name__)
+ self.assertEqual('170EF19C0FA0491DFCEA6E1FB0941670B80506E1', f.fingerprint)
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_microdescriptors(self, files_mock, download_mock):
+ with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/microdescs/microdescs-2019-05.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_microdescriptors())
+ self.assertEqual(3, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('Microdescriptor', type(f).__name__)
+ self.assertEqual(['ed25519'], list(f.identifiers.keys()))
+
+ @patch('stem.util.connection.download')
+ @patch('stem.descriptor.collector.CollecTor.files')
+ def test_reading_consensus(self, files_mock, download_mock):
+ with open(get_resource('collector/consensuses-2018-06-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ files_mock.return_value = [stem.descriptor.collector.File('archive/relay-descriptors/consensuses/consensuses-2018-06.tar', 12345, '2016-09-04 09:21')]
+
+ descriptors = list(stem.descriptor.collector.get_consensus())
+ self.assertEqual(243, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('RouterStatusEntryV3', type(f).__name__)
+ self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint)
+
+ # this archive shouldn't have any v2 or microdescriptor consensus data
+
+ self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(version = 2))))
+ self.assertEqual(0, len(list(stem.descriptor.collector.get_consensus(microdescriptor = True))))
+
+ # but the microdescriptor archive *does* have microdescriptor consensuses
+
+ with open(get_resource('collector/microdescs-2019-05-cropped.tar'), 'rb') as archive:
+ download_mock.return_value = archive.read()
+
+ descriptors = list(stem.descriptor.collector.get_consensus(microdescriptor = True))
+ self.assertEqual(556, len(descriptors))
+
+ f = descriptors[0]
+ self.assertEqual('RouterStatusEntryMicroV3', type(f).__name__)
+ self.assertEqual('000A10D43011EA4928A35F610405F92B4433B4DC', f.fingerprint)
diff --git a/test/unit/descriptor/data/__init__.py b/test/unit/descriptor/data/__init__.py
index 376864fd..8bc640c6 100644
--- a/test/unit/descriptor/data/__init__.py
+++ b/test/unit/descriptor/data/__init__.py
@@ -3,5 +3,5 @@ Test data for test.unit.descriptor tests.
"""
__all__ = [
- 'collector_index',
+ 'collector',
]
diff --git a/test/unit/descriptor/data/collector/__init__.py b/test/unit/descriptor/data/collector/__init__.py
new file mode 100644
index 00000000..fe90c9fb
--- /dev/null
+++ b/test/unit/descriptor/data/collector/__init__.py
@@ -0,0 +1,7 @@
+"""
+Test data for test.unit.descriptor's collector tests.
+"""
+
+__all__ = [
+ 'index',
+]
diff --git a/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar
new file mode 100644
index 00000000..0728e595
Binary files /dev/null and b/test/unit/descriptor/data/collector/consensuses-2018-06-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar
new file mode 100644
index 00000000..21227286
Binary files /dev/null and b/test/unit/descriptor/data/collector/extra-infos-2019-04-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector_index.json b/test/unit/descriptor/data/collector/index.json
similarity index 100%
rename from test/unit/descriptor/data/collector_index.json
rename to test/unit/descriptor/data/collector/index.json
diff --git a/test/unit/descriptor/data/collector_index.py b/test/unit/descriptor/data/collector/index.py
similarity index 100%
rename from test/unit/descriptor/data/collector_index.py
rename to test/unit/descriptor/data/collector/index.py
diff --git a/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar
new file mode 100644
index 00000000..789774c2
Binary files /dev/null and b/test/unit/descriptor/data/collector/microdescs-2019-05-cropped.tar differ
diff --git a/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar
new file mode 100644
index 00000000..4ffe1434
Binary files /dev/null and b/test/unit/descriptor/data/collector/server-descriptors-2005-12-cropped.tar differ
More information about the tor-commits
mailing list