[tor-commits] [nyx/master] Run static checks in the background

[atagar at torproject.org]

commit c03d05abdec29eebba2d3ed0f15c7e5f46aa658b
Author: Damian Johnson <atagar at torproject.org>
Date:   Wed Aug 16 07:46:03 2017 -0700

    Run static checks in the background
    
    Stem got quite a substantial speedup in its tests by running long-running tasks
    in the background. Nyx is a much smaller test suite so not much to gain, but
    might as well improve things a tad here too.
    
    Running pyflakes and pycodestyle in the background. On my laptop this drops the
    test runtime from 3.8s to 2.4s (37% faster).
---
 run_tests.py | 41 +++++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/run_tests.py b/run_tests.py
index 2b76561..e115552 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -9,6 +9,7 @@ render to the screen.
 """
 
 import os
+import multiprocessing
 import unittest
 
 import stem.util.conf
@@ -27,6 +28,12 @@ SRC_PATHS = [os.path.join(NYX_BASE, path) for path in (
 )]
 
 
+def _run_wrapper(conn, runner, args):
+  os.nice(15)
+  conn.send(runner(*args) if args else runner())
+  conn.close()
+
+
 @nyx.uses_settings
 def main():
   nyx.TESTING = True
@@ -38,7 +45,20 @@ def main():
   for path in orphaned_pyc:
     print('Deleted orphaned pyc file: %s' % path)
 
-  tests = unittest.defaultTestLoader.discover('test', pattern='*.py')
+  pyflakes_task, pyflakes_pipe = None, None
+  pycodestyle_task, pycodestyle_pipe = None, None
+
+  if stem.util.test_tools.is_pyflakes_available():
+    pyflakes_pipe, child_pipe = multiprocessing.Pipe()
+    pyflakes_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.pyflakes_issues, (SRC_PATHS,)))
+    pyflakes_task.start()
+
+  if stem.util.test_tools.is_pep8_available():
+    pycodestyle_pipe, child_pipe = multiprocessing.Pipe()
+    pycodestyle_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.stylistic_issues, (SRC_PATHS, True, True, True)))
+    pycodestyle_task.start()
+
+  tests = unittest.defaultTestLoader.discover('test', pattern = '*.py')
   test_runner = unittest.TextTestRunner()
   test_runner.run(tests)
 
@@ -46,22 +66,19 @@ def main():
 
   static_check_issues = {}
 
-  if stem.util.test_tools.is_pyflakes_available():
-    pyflakes_issues = stem.util.test_tools.pyflakes_issues(SRC_PATHS)
+  if pyflakes_task:
+    pyflakes_issues = pyflakes_pipe.recv()
+    pyflakes_task.join()
 
     for path, issues in pyflakes_issues.items():
       for issue in issues:
         static_check_issues.setdefault(path, []).append(issue)
 
-  if stem.util.test_tools.is_pep8_available():
-    pep8_issues = stem.util.test_tools.stylistic_issues(
-      SRC_PATHS,
-      check_newlines = True,
-      check_exception_keyword = True,
-      prefer_single_quotes = True,
-    )
-
-    for path, issues in pep8_issues.items():
+  if pycodestyle_task:
+    pycodestyle_issues = pycodestyle_pipe.recv()
+    pycodestyle_task.join()
+
+    for path, issues in pycodestyle_issues.items():
       for issue in issues:
         static_check_issues.setdefault(path, []).append(issue)