[tor-commits] [stem/master] Change tail() to be a generator

[atagar at torproject.org]

commit 0da547cf8119307f436e642148427e8e58328c92
Author: Damian Johnson <atagar at torproject.org>
Date:   Fri Apr 3 09:26:29 2015 -0700

    Change tail() to be a generator
    
    Switching tail() to be a generator so it can provide constant memory usage, and
    allow the caller to abort reading content at any time.
---
 stem/util/system.py      |   30 +++++++++++++++++++-----------
 test/unit/util/system.py |   16 ++++++++--------
 2 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/stem/util/system.py b/stem/util/system.py
index 56f3da5..f33d8b7 100644
--- a/stem/util/system.py
+++ b/stem/util/system.py
@@ -750,19 +750,27 @@ def start_time(pid):
 
 def tail(target, lines = None):
   """
-  Provides the last lines from a file, similar to 'tail -n 50 /tmp/my_log'.
+  Provides lines of a file starting with the end. For instance,
+  'tail -n 50 /tmp/my_log' could be done with...
+
+  ::
+
+    reversed(list(tail('/tmp/my_log', 50)))
 
   :param str,file target: path or file object to read from
   :param int lines: number of lines to read
 
-  :returns: **list** of lines the file ends with
+  :returns: **generator** that reads lines, starting with the end
 
   :raises: **IOError** if unable to read the file
   """
 
   if isinstance(target, str):
     with open(target) as target_file:
-      return tail(target_file, lines)
+      for line in tail(target_file, lines):
+        yield line
+
+      return
 
   if lines is None:
     lines = sys.maxint
@@ -774,26 +782,26 @@ def tail(target, lines = None):
   block_end_byte = target.tell()
   lines_left = lines
   block_number = -1
-  blocks = []  # blocks of size BLOCK_SIZE, in reverse order
+  content = ''
 
   while lines_left > 0 and block_end_byte > 0:
     if (block_end_byte - BLOCK_SIZE > 0):
       # read the last block we haven't yet read
       target.seek(block_number * BLOCK_SIZE, 2)
-      blocks.insert(0, target.read(BLOCK_SIZE))
+      content, completed_lines = (target.read(BLOCK_SIZE) + content).split('\n', 1)
     else:
       # reached the start of the file, just read what's left
       target.seek(0, 0)
-      blocks.insert(0, target.read(block_end_byte))
+      completed_lines = target.read(block_end_byte) + content
+
+    for line in reversed(completed_lines.splitlines()):
+      if lines_left > 0:
+        lines_left -= 1
+        yield line
 
-    lines_found = blocks[-1].count('\n')
-    lines_left -= lines_found
     block_end_byte -= BLOCK_SIZE
     block_number -= 1
 
-  text = ''.join(blocks)
-  return text.splitlines()[-lines:]
-
 
 def bsd_jail_id(pid):
   """
diff --git a/test/unit/util/system.py b/test/unit/util/system.py
index e3de4d8..dd1e30f 100644
--- a/test/unit/util/system.py
+++ b/test/unit/util/system.py
@@ -383,20 +383,20 @@ class TestSystem(unittest.TestCase):
     # by file handle
 
     with open(path) as riddle_file:
-      self.assertEqual(['  both the wicked and sweet.'], system.tail(riddle_file, 1))
+      self.assertEqual(['  both the wicked and sweet.'], list(system.tail(riddle_file, 1)))
 
-    self.assertEqual([], system.tail(path, 0))
-    self.assertEqual(['  both the wicked and sweet.'], system.tail(path, 1))
-    self.assertEqual(["but I'm with people you meet", '  both the wicked and sweet.'], system.tail(path, 2))
+    self.assertEqual([], list(system.tail(path, 0)))
+    self.assertEqual(['  both the wicked and sweet.'], list(system.tail(path, 1)))
+    self.assertEqual(['  both the wicked and sweet.', "but I'm with people you meet"], list(system.tail(path, 2)))
 
-    self.assertEqual(14, len(system.tail(path)))
-    self.assertEqual(14, len(system.tail(path, 200)))
+    self.assertEqual(14, len(list(system.tail(path))))
+    self.assertEqual(14, len(list(system.tail(path, 200))))
 
-    self.assertRaises(IOError, system.tail, '/path/doesnt/exist')
+    self.assertRaises(IOError, list, system.tail('/path/doesnt/exist'))
 
     fd, temp_path = tempfile.mkstemp()
     os.chmod(temp_path, 0o077)  # remove read permissions
-    self.assertRaises(IOError, system.tail, temp_path)
+    self.assertRaises(IOError, list, system.tail(temp_path))
     os.close(fd)
     os.remove(temp_path)