[tor-commits] [exonerator/master] Switch to using metrics-lib's DescriptorCollector.

[karsten at torproject.org]

commit fa377c3535e9a0fd4ed6bcd051b89f01ebf30739
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Thu May 28 10:36:49 2015 +0200

    Switch to using metrics-lib's DescriptorCollector.
---
 .gitmodules                                               |    3 +++
 build.xml                                                 |    9 ++++++++-
 deps/metrics-lib                                          |    1 +
 run-exonerator.sh                                         |    3 ---
 .../torproject/exonerator/ExoneraTorDatabaseImporter.java |   13 +++++++++++++
 5 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..c6b6388
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "deps/metrics-lib"]
+	path = deps/metrics-lib
+	url = https://git.torproject.org/metrics-lib
diff --git a/build.xml b/build.xml
index b6a1e53..3064bf8 100644
--- a/build.xml
+++ b/build.xml
@@ -11,6 +11,9 @@
   <property name="warfile" value="exonerator.war"/>
   <path id="classpath">
     <pathelement path="${classes}"/>
+    <fileset dir="deps/metrics-lib">
+      <include name="descriptor.jar"/>
+    </fileset>
     <fileset dir="/usr/share/java">
       <include name="commons-codec.jar"/>
       <include name="postgresql-jdbc3.jar"/>
@@ -25,8 +28,12 @@
     <mkdir dir="${classes}"/>
   </target>
 
+  <target name="metrics-lib">
+    <ant dir="deps/metrics-lib"/>
+  </target>
+
   <!-- Compile all servlets and plain Java classes. -->
-  <target name="compile" depends="init">
+  <target name="compile" depends="metrics-lib, init">
     <javac destdir="${classes}"
            srcdir="${sources}"
            source="1.5"
diff --git a/deps/metrics-lib b/deps/metrics-lib
new file mode 160000
index 0000000..bd18dd0
--- /dev/null
+++ b/deps/metrics-lib
@@ -0,0 +1 @@
+Subproject commit bd18dd0d2f788d1d4ee7acbac5032e97264bb855
diff --git a/run-exonerator.sh b/run-exonerator.sh
index 6cd34bf..d090ec9 100755
--- a/run-exonerator.sh
+++ b/run-exonerator.sh
@@ -1,6 +1,3 @@
 #!/bin/bash
-rsync -arz --delete metrics.torproject.org::metrics-recent/relay-descriptors/consensuses exonerator-import/
-rsync -arz --delete metrics.torproject.org::metrics-recent/relay-descriptors/server-descriptors exonerator-import/
-rsync -arz --delete metrics.torproject.org::metrics-recent/exit-lists exonerator-import/
 ant run | grep "\[java\]"
 
diff --git a/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java b/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java
index 348c04c..6111888 100644
--- a/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java
+++ b/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java
@@ -32,6 +32,8 @@ import java.util.TimeZone;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.descriptor.DescriptorCollector;
+import org.torproject.descriptor.DescriptorSourceFactory;
 
 /* Import Tor descriptors into the ExoneraTor database. */
 public class ExoneraTorDatabaseImporter {
@@ -42,6 +44,7 @@ public class ExoneraTorDatabaseImporter {
     openDatabaseConnection();
     prepareDatabaseStatements();
     createLockFile();
+    fetchDescriptors();
     readImportHistoryToMemory();
     parseDescriptors();
     writeImportHistoryToDisk();
@@ -150,6 +153,16 @@ public class ExoneraTorDatabaseImporter {
     }
   }
 
+  /* Fetch recent descriptors from CollecTor. */
+  private static void fetchDescriptors() {
+    DescriptorCollector collector =
+        DescriptorSourceFactory.createDescriptorCollector();
+    collector.collectDescriptors("https://collector.torproject.org",
+        new String[] { "/recent/relay-descriptors/consensuses/",
+        "/recent/relay-descriptors/server-descriptors/",
+        "/recent/exit-lists/" }, 0L, new File(importDirString), true);
+  }
+
   /* Last and next parse histories containing paths of parsed files and
    * last modified times. */
   private static Map<String, Long>