[tor-commits] [onionoo/master] Switch to parsing concatenated descriptors from rsync recent.

[karsten at torproject.org]

commit 2d1c226bfcd160ff91a98b36cbe2ac0cb3a83a79
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Sun May 25 08:34:33 2014 +0200

    Switch to parsing concatenated descriptors from rsync recent.
---
 bin/update.sh                                    |    2 +-
 build.xml                                        |    2 +-
 deps/metrics-lib                                 |    2 +-
 src/org/torproject/onionoo/DescriptorSource.java |   17 +++++++++++++----
 4 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/bin/update.sh b/bin/update.sh
index 0d48e59..9f1424f 100755
--- a/bin/update.sh
+++ b/bin/update.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
-rsync -az --delete metrics.torproject.org::'metrics-recent/relay-descriptors/consensuses/ metrics-recent/relay-descriptors/server-descriptors/ metrics-recent/relay-descriptors/extra-infos/ metrics-recent/bridge-descriptors/statuses/ metrics-recent/bridge-descriptors/server-descriptors/ metrics-recent/bridge-descriptors/extra-infos/ metrics-recent/bridge-pool-assignments/ metrics-recent/exit-lists' in
+rsync -az --delete --exclude 'relay-descriptors/votes' --exclude 'relay-descriptors/microdescs' --exclude 'relay-descriptors/server-descriptors' --exclude 'relay-descriptors/extra-infos' --exclude 'bridge-descriptors/server-descriptors' --exclude 'bridge-descriptors/extra-infos' --exclude 'torperf' metrics.torproject.org::metrics-recent in
 ant run >> log
 
diff --git a/build.xml b/build.xml
index e772da4..500204d 100644
--- a/build.xml
+++ b/build.xml
@@ -92,7 +92,7 @@
   </target>
   <target name="run" depends="compile">
     <java fork="true"
-          maxmemory="2048m"
+          maxmemory="4g"
           classname="org.torproject.onionoo.Main">
       <classpath refid="classpath"/>
     </java>
diff --git a/deps/metrics-lib b/deps/metrics-lib
index 3e60ccd..a472d7f 160000
--- a/deps/metrics-lib
+++ b/deps/metrics-lib
@@ -1 +1 @@
-Subproject commit 3e60ccdaaba598cabb7281d45f9a415299b8e3e3
+Subproject commit a472d7f3425feec9b9bed552b05af33dab20fee1
diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java
index f4ff2fc..0692854 100644
--- a/src/org/torproject/onionoo/DescriptorSource.java
+++ b/src/org/torproject/onionoo/DescriptorSource.java
@@ -106,24 +106,29 @@ class DescriptorQueue {
 
   public void addDirectory(DescriptorType descriptorType) {
     String directoryName = null;
+    int maxDescriptorFilesInQueue = -1;
     switch (descriptorType) {
     case RELAY_CONSENSUSES:
       directoryName = "relay-descriptors/consensuses";
       break;
     case RELAY_SERVER_DESCRIPTORS:
-      directoryName = "relay-descriptors/server-descriptors";
+      directoryName = "relay-descriptors/server-descriptors-cat";
+      maxDescriptorFilesInQueue = 1;
       break;
     case RELAY_EXTRA_INFOS:
-      directoryName = "relay-descriptors/extra-infos";
+      directoryName = "relay-descriptors/extra-infos-cat";
+      maxDescriptorFilesInQueue = 1;
       break;
     case BRIDGE_STATUSES:
       directoryName = "bridge-descriptors/statuses";
       break;
     case BRIDGE_SERVER_DESCRIPTORS:
-      directoryName = "bridge-descriptors/server-descriptors";
+      directoryName = "bridge-descriptors/server-descriptors-cat";
+      maxDescriptorFilesInQueue = 1;
       break;
     case BRIDGE_EXTRA_INFOS:
-      directoryName = "bridge-descriptors/extra-infos";
+      directoryName = "bridge-descriptors/extra-infos-cat";
+      maxDescriptorFilesInQueue = 1;
       break;
     case BRIDGE_POOL_ASSIGNMENTS:
       directoryName = "bridge-pool-assignments";
@@ -139,6 +144,10 @@ class DescriptorQueue {
     File directory = new File(this.inDir, directoryName);
     if (directory.exists() && directory.isDirectory()) {
       this.descriptorReader.addDirectory(directory);
+      if (maxDescriptorFilesInQueue >= 1) {
+        this.descriptorReader.setMaxDescriptorFilesInQueue(
+            maxDescriptorFilesInQueue);
+      }
     } else {
       System.err.println("Directory " + directory.getAbsolutePath()
           + " either does not exist or is not a directory.  Not adding "