[tor-commits] [torflow/master] Double the scanners, double the tors
mikeperry at torproject.org
mikeperry at torproject.org
Mon Jun 1 05:37:44 UTC 2015
commit 94dff967fd457c4922fe7838f5f5ef4c7b17eefe
Author: Tom Ritter <tom at ritter.vg>
Date: Thu May 21 13:01:07 2015 -0500
Double the scanners, double the tors
Also update aggregate.py to work with a two-tor system
---
NetworkScanners/BwAuthority/aggregate.py | 6 ++--
.../BwAuthority/data/scanner.1/bwauthority.cfg | 4 +--
.../BwAuthority/data/scanner.2/bwauthority.cfg | 6 ++--
.../BwAuthority/data/scanner.3/bwauthority.cfg | 6 ++--
.../BwAuthority/data/scanner.4/bwauthority.cfg | 6 ++--
.../BwAuthority/data/scanner.5/bwauthority.cfg | 25 ++++++++--------
.../BwAuthority/data/scanner.6/bwauthority.cfg | 30 ++++++++++++++++++++
.../BwAuthority/data/scanner.7/bwauthority.cfg | 30 ++++++++++++++++++++
.../BwAuthority/data/scanner.8/bwauthority.cfg | 30 ++++++++++++++++++++
.../BwAuthority/data/scanner.9/bwauthority.cfg | 30 ++++++++++++++++++++
.../data/scanner.9/scan-data/.gitignore | 2 ++
NetworkScanners/BwAuthority/data/tor.1/torrc | 18 ++++++++++++
NetworkScanners/BwAuthority/data/tor.2/torrc | 18 ++++++++++++
NetworkScanners/BwAuthority/data/tor/torrc | 18 ------------
NetworkScanners/BwAuthority/run_scan.sh | 29 +++++++++++--------
15 files changed, 203 insertions(+), 55 deletions(-)
diff --git a/NetworkScanners/BwAuthority/aggregate.py b/NetworkScanners/BwAuthority/aggregate.py
index 49ad3cc..6d35dba 100755
--- a/NetworkScanners/BwAuthority/aggregate.py
+++ b/NetworkScanners/BwAuthority/aggregate.py
@@ -362,7 +362,8 @@ def write_file_list(datadir):
def main(argv):
TorUtil.read_config(argv[1]+"/scanner.1/bwauthority.cfg")
- TorUtil.loglevel = "NOTICE"
+ TorUtil.loglevel = "DEBUG"
+ TorUtil.logfile = "data/aggregate-debug.log"
(branch, head) = TorUtil.get_git_version(PATH_TO_TORFLOW_REPO)
plog('INFO', 'TorFlow Version: %s' % branch+' '+head)
@@ -373,7 +374,7 @@ def main(argv):
s.connect((TorUtil.control_host,TorUtil.control_port))
c = TorCtl.Connection(s)
c.debug(file(argv[1]+"/aggregate-control.log", "w", buffering=0))
- c.authenticate_cookie(file(argv[1]+"/tor/control_auth_cookie",
+ c.authenticate_cookie(file(argv[1]+"/tor.1/control_auth_cookie",
"r"))
ns_list = c.get_network_status()
@@ -847,6 +848,7 @@ def main(argv):
plog("INFO", "Avg network +pid_error=+"+str(sum(map(lambda n: n.pid_error, p_nodes))/len(p_nodes)))
plog("INFO", "Avg network -pid_error="+str(sum(map(lambda n: n.pid_error, n_nodes))/len(n_nodes)))
+
plog("INFO",
"Measured "+str(measured_pct) +"% of all tor nodes ("
+str(measured_bw_pct)+"% of previous consensus bw).")
diff --git a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
index 7f81a0b..ad9a4cd 100644
--- a/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.1/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.1/bwauthority.pid
db_url =
#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan1
#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan1
-tor_dir = ./data/tor
+tor_dir = ./data/tor.1
start_pct = 0
-stop_pct = 10
+stop_pct = 12
save_every = 0
nodes_per_slice = 50
circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
index 28cfaa6..3e94e5e 100644
--- a/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.2/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.2/bwauthority.pid
db_url =
#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan2
#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan2
-tor_dir = ./data/tor
-start_pct = 10
-stop_pct = 40
+tor_dir = ./data/tor.1
+start_pct = 12
+stop_pct = 25
save_every = 0
nodes_per_slice = 50
circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
index 7b71aa8..71c3285 100644
--- a/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.3/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.3/bwauthority.pid
db_url =
#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan3
#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan3
-tor_dir = ./data/tor
-start_pct = 30
-stop_pct = 60
+tor_dir = ./data/tor.1
+start_pct = 25
+stop_pct = 37
save_every = 0
nodes_per_slice = 50
circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
index 0dfea36..9a676ec 100644
--- a/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.4/bwauthority.cfg
@@ -16,9 +16,9 @@ pid_file = ./data/scanner.4/bwauthority.pid
db_url =
#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4
#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4
-tor_dir = ./data/tor
-start_pct = 60
-stop_pct = 100
+tor_dir = ./data/tor.1
+start_pct = 37
+stop_pct = 50
save_every = 0
nodes_per_slice = 50
circs_per_node = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
index 468a62e..1acb1a2 100644
--- a/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
+++ b/NetworkScanners/BwAuthority/data/scanner.5/bwauthority.cfg
@@ -1,30 +1,31 @@
[TorCtl]
loglevel=WARN
tor_host = 127.0.0.1
-tor_port = 9110
+tor_port = 9120
control_host = 127.0.0.1
-control_port = 9111
+control_port = 9121
control_pass =
# XXX: Unused
meta_host = 127.0.0.1
-meta_port = 9112
+meta_port = 9122
[BwAuthority]
out_dir = ./data/scanner.5/scan-data
pid_file = ./data/scanner.5/bwauthority.pid
# if db_url is unset bwauthority will default to sqlite
-db_url =
-#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan4
-#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan4
-tor_dir = ./data/tor
-start_pct = 0
-stop_pct = 100
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan5
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan5
+tor_dir = ./data/tor.2
+start_pct = 50
+stop_pct = 62
save_every = 0
nodes_per_slice = 50
circs_per_node = 5
min_streams = 1
-max_fetch_time = 360
+max_fetch_time = 300
sleep_start = 01:30
sleep_stop = 01:30
-only_unmeasured = 1
-min_unmeasured = 5
+only_unmeasured = 0
+min_unmeasured = 0
+
diff --git a/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg
new file mode 100755
index 0000000..5bc3616
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.6/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.6/scan-data
+pid_file = ./data/scanner.6/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan6
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan6
+tor_dir = ./data/tor.2
+start_pct = 62
+stop_pct = 75
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg
new file mode 100755
index 0000000..2898b80
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.7/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.7/scan-data
+pid_file = ./data/scanner.7/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan7
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan7
+tor_dir = ./data/tor.2
+start_pct = 75
+stop_pct = 87
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg
new file mode 100755
index 0000000..56a5cfa
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.8/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.8/scan-data
+pid_file = ./data/scanner.8/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan8
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan8
+tor_dir = ./data/tor.2
+start_pct = 87
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 300
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 0
+min_unmeasured = 0
diff --git a/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg
new file mode 100644
index 0000000..7e49dbb
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.9/bwauthority.cfg
@@ -0,0 +1,30 @@
+[TorCtl]
+loglevel=WARN
+tor_host = 127.0.0.1
+tor_port = 9120
+control_host = 127.0.0.1
+control_port = 9121
+control_pass =
+# XXX: Unused
+meta_host = 127.0.0.1
+meta_port = 9122
+
+[BwAuthority]
+out_dir = ./data/scanner.9/scan-data
+pid_file = ./data/scanner.9/bwauthority.pid
+# if db_url is unset bwauthority will default to sqlite
+db_url =
+#db_url = mysql+mysqldb://bwscanner:password@127.0.0.1/BwScan9
+#db_url = postgresql://bwscanner:password@127.0.0.1/BwScan9
+tor_dir = ./data/tor.2
+start_pct = 0
+stop_pct = 100
+save_every = 0
+nodes_per_slice = 50
+circs_per_node = 5
+min_streams = 1
+max_fetch_time = 360
+sleep_start = 01:30
+sleep_stop = 01:30
+only_unmeasured = 1
+min_unmeasured = 5
diff --git a/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore
new file mode 100644
index 0000000..20f86df
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/scanner.9/scan-data/.gitignore
@@ -0,0 +1,2 @@
+bws-*
+sql-*
\ No newline at end of file
diff --git a/NetworkScanners/BwAuthority/data/tor.1/torrc b/NetworkScanners/BwAuthority/data/tor.1/torrc
new file mode 100644
index 0000000..933141e
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/tor.1/torrc
@@ -0,0 +1,18 @@
+## Configuration file for metatroller on alternate prot
+SocksPort 9110 # what port to advertise for application connections
+ControlPort 9111
+SafeLogging 0
+Log notice file ./data/tor.1/tor.log
+DataDirectory ./data/tor.1
+PidFile ./data/tor.1/tor.pid
+CookieAuthentication 1
+FetchUselessDescriptors 1
+__LeaveStreamsUnattached 1
+
+# Bad idea? Too much consensus update activity?
+FetchDirInfoExtraEarly 1
+FetchDirInfoEarly 1
+
+# Use CREATE cells for first hop, so we can still measure CPU overload via
+# circ failure
+FastFirstHopPK 0
diff --git a/NetworkScanners/BwAuthority/data/tor.2/torrc b/NetworkScanners/BwAuthority/data/tor.2/torrc
new file mode 100755
index 0000000..209781f
--- /dev/null
+++ b/NetworkScanners/BwAuthority/data/tor.2/torrc
@@ -0,0 +1,18 @@
+## Configuration file for metatroller on alternate prot
+SocksPort 9120 # what port to advertise for application connections
+ControlPort 9121
+SafeLogging 0
+Log notice file ./data/tor.2/tor.log
+DataDirectory ./data/tor.2
+PidFile ./data/tor.2/tor.pid
+CookieAuthentication 1
+FetchUselessDescriptors 1
+__LeaveStreamsUnattached 1
+
+# Bad idea? Too much consensus update activity?
+FetchDirInfoExtraEarly 1
+FetchDirInfoEarly 1
+
+# Use CREATE cells for first hop, so we can still measure CPU overload via
+# circ failure
+FastFirstHopPK 0
diff --git a/NetworkScanners/BwAuthority/data/tor/torrc b/NetworkScanners/BwAuthority/data/tor/torrc
deleted file mode 100644
index c64273b..0000000
--- a/NetworkScanners/BwAuthority/data/tor/torrc
+++ /dev/null
@@ -1,18 +0,0 @@
-## Configuration file for metatroller on alternate prot
-SocksPort 9110 # what port to advertise for application connections
-ControlPort 9111
-SafeLogging 0
-Log notice file ./data/tor/tor.log
-DataDirectory ./data/tor
-PidFile ./data/tor/tor.pid
-CookieAuthentication 1
-FetchUselessDescriptors 1
-__LeaveStreamsUnattached 1
-
-# Bad idea? Too much consensus update activity?
-FetchDirInfoExtraEarly 1
-FetchDirInfoEarly 1
-
-# Use CREATE cells for first hop, so we can still measure CPU overload via
-# circ failure
-FastFirstHopPK 0
diff --git a/NetworkScanners/BwAuthority/run_scan.sh b/NetworkScanners/BwAuthority/run_scan.sh
index 8a71b1d..908b76e 100755
--- a/NetworkScanners/BwAuthority/run_scan.sh
+++ b/NetworkScanners/BwAuthority/run_scan.sh
@@ -1,7 +1,9 @@
#!/bin/sh
-# Number of scanners to run.
-SCANNER_COUNT=5
+# Number of applications to run.
+SCANNERS_PER_TOR_COUNT=4
+TOR_COUNT=2
+SCANNER_COUNT=$(($SCANNERS_PER_TOR_COUNT * $TOR_COUNT + 1))
# This tor must have the w status line fix as well as the stream bw fix
# Ie git master or 0.2.2.x
@@ -19,15 +21,17 @@ for n in `seq $SCANNER_COUNT`; do
done
KILLED_TOR=false
-if [ -f "./data/tor/tor.pid" ]; then
- PID=`cat ./data/tor/tor.pid`
- if kill -0 "$PID" 2>/dev/null; then # it is a running process and we may send signals to it
- kill $PID
- if [ $? -eq 0 ]; then
- KILLED_TOR=true
+for n in `seq $TOR_COUNT`; do
+ PIDFILE=./data/tor.${n}/tor.pid
+ if [ -f $PIDFILE ]; then
+ if kill -0 `head -1 $PIDFILE` 2>/dev/null; then # it is a running process and we may send signals to it
+ kill `head -1 $PIDFILE`
+ if [ $? -eq 0 ]; then
+ KILLED_TOR=true
+ fi
fi
fi
-fi
+done
sleep 5
@@ -35,9 +39,10 @@ sleep 5
# A more accurate resume could be implemented in bwauthority.py
find data/scanner.* -name .svn -prune -o -type f -a ! -name '*-done-*' -a ! -name bwauthority.cfg -a ! -name .gitignore -exec rm {} +
-rm -f ./data/tor/tor.log
-
-$TOR_EXE -f ./data/tor/torrc &
+for n in `seq $TOR_COUNT`; do
+ rm -f ./data/tor.${n}/tor.log
+ $TOR_EXE -f ./data/tor.${n}/torrc &
+done
# If this is a fresh start, we should allow the tors time to download
# new descriptors.
More information about the tor-commits
mailing list