[or-cvs] r19414: {torflow} Make metatroller and speedracer use new SQL support. (in torflow/trunk: . NetworkScanners)

[mikeperry at seul.org]

Author: mikeperry
Date: 2009-05-04 02:58:15 -0400 (Mon, 04 May 2009)
New Revision: 19414

Modified:
   torflow/trunk/NetworkScanners/speedracer.py
   torflow/trunk/metatroller.py
Log:

Make metatroller and speedracer use new SQL support.



Modified: torflow/trunk/NetworkScanners/speedracer.py
===================================================================
--- torflow/trunk/NetworkScanners/speedracer.py	2009-05-04 06:57:13 UTC (rev 19413)
+++ torflow/trunk/NetworkScanners/speedracer.py	2009-05-04 06:58:15 UTC (rev 19414)
@@ -34,7 +34,7 @@
 pct_step = 3
 # Number of fetches per slice:
 count = 250
-save_every = 10
+save_every = 2
 
 class MetatrollerException(Exception):
     "Metatroller does not accept this command."
@@ -124,11 +124,12 @@
         else:
             plog('DEBUG', str(skip) + '-' + str(pct) + '% circuit build+fetch failed for ' + str(build_exit))
 
-        if (successful % save_every) == 0:
+        if successful and (successful % save_every) == 0:
           race_time = strftime("20%y-%m-%d-%H:%M:%S")
           meta.send_command_and_check('CLOSEALLCIRCS')
           meta.send_command_and_check('SAVESTATS '+os.getcwd()+'/data/speedraces/stats-'+str(skip)+':'+str(pct)+"-"+str(successful)+"-"+race_time)
           meta.send_command_and_check('SAVERATIOS '+os.getcwd()+'/data/speedraces/ratios-'+str(skip)+':'+str(pct)+"-"+str(successful)+"-"+race_time)
+          meta.send_command_and_check('SAVESQL '+os.getcwd()+'/data/speedraces/sql-'+str(skip)+':'+str(pct)+"-"+str(successful)+"-"+race_time)
           meta.send_command_and_check('COMMIT')
 
     plog('INFO', str(skip) + '-' + str(pct) + '% ' + str(count) + ' fetches took ' + str(attempt) + ' tries.')
@@ -148,6 +149,7 @@
         
     # configure metatroller
     commands = [
+        'SQLSUPPORT sqlite:///'+os.getcwd()+'/data/speedraces/speedracer.sqlite',
         'PATHLEN 2',
         'UNIFORM 1',
         'ORDEREXITS 0',
@@ -176,6 +178,7 @@
           meta.send_command_and_check('CLOSEALLCIRCS')
           meta.send_command_and_check('SAVESTATS '+os.getcwd()+'/data/speedraces/stats-'+str(pct) + ':' + str(pct + pct_step)+"-"+str(count)+"-"+strftime("20%y-%m-%d-%H:%M:%S"))
           meta.send_command_and_check('SAVERATIOS '+os.getcwd()+'/data/speedraces/ratios-'+str(pct) + ':' + str(pct + pct_step)+"-"+str(count)+"-"+strftime("20%y-%m-%d-%H:%M:%S"))
+          meta.send_command_and_check('SAVESQL '+os.getcwd()+'/data/speedraces/sql-'+str(pct) + ':' + str(pct + pct_step)+"-"+str(count)+"-"+strftime("20%y-%m-%d-%H:%M:%S"))
           plog('DEBUG', 'Wrote stats')
           pct += pct_step
           meta.send_command_and_check('COMMIT')

Modified: torflow/trunk/metatroller.py
===================================================================
--- torflow/trunk/metatroller.py	2009-05-04 06:57:13 UTC (rev 19413)
+++ torflow/trunk/metatroller.py	2009-05-04 06:58:15 UTC (rev 19414)
@@ -65,6 +65,9 @@
 def commandloop(s, c, h):
   "The main metatroller listener loop"
   s.write("220 Welcome to the Tor Metatroller "+mt_version+"! Try HELP for Info\r\n\r\n")
+
+  percent_skip=__selmgr.percent_skip
+  percent_fast=__selmgr.percent_fast
   while 1:
     buf = s.readline()
     if not buf: break
@@ -215,6 +218,19 @@
         s.write("250 OK\r\n")
       except ValueError:
         s.write("510 Integer expected\r\n")
+    elif command == "SQLSUPPORT":
+     try:
+        if arg:
+          plog("DEBUG", "Got sqlite: "+arg)
+          use_db = arg
+          from TorCtl import SQLSupport
+          SQLSupport.setup_db(use_db, True)
+          h.add_event_listener(SQLSupport.ConsensusTrackerListener())
+          h.add_event_listener(SQLSupport.StreamListener())
+          plog("DEBUG", "Did sqlite: "+arg)
+          s.write("250 OK\r\n")
+     except ValueError:
+       s.write("510 database expected\r\n")
     elif command == "CLOSEALLCIRCS":
       def notlambda(this): this.close_all_circuits()
       h.schedule_immediate(notlambda)
@@ -231,6 +247,24 @@
       def notlambda(this): this.write_ratios(rfilename)
       h.schedule_low_prio(notlambda)
       s.write("250 OK\r\n")
+    elif command == "SAVESQL":
+      # TODO: Use threading conditions more. Maybe even get some
+      # better status reporting than always blindly printing OK.
+
+      if arg: rfilename = arg
+      else: rfilename="./data/stats/sql-"+time.strftime("20%y-%m-%d-%H:%M:%S")
+      cond = threading.Condition() 
+      def notlambda(h):
+        cond.acquire()
+        SQLSupport.RouterStats.write_stats(file(rfilename, "w"),
+                             percent_skip, percent_fast, recompute=True)
+        cond.notify()
+        cond.release()
+      cond.acquire()
+      h.schedule_low_prio(notlambda)
+      cond.wait()
+      cond.release()
+      s.write("250 OK\r\n")
     elif command == "RESETSTATS":
       plog("DEBUG", "Got resetstats")
       def notlambda(this): this.reset_stats()