[tor-commits] [metrics-db/master] Run five data-processing modules in parallel.

[karsten at torproject.org]

commit 7db75dbedf2fc77a3855e4fdff051bcbdcccac5e
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Fri Oct 26 15:13:21 2012 -0400

    Run five data-processing modules in parallel.
---
 .../db/bridgedescs/SanitizedBridgesWriter.java     |   19 ++++++++++++-------
 .../BridgePoolAssignmentsProcessor.java            |    8 +++++++-
 .../ernie/db/exitlists/ExitListDownloader.java     |    6 +++++-
 src/org/torproject/ernie/db/main/Main.java         |   14 +++++---------
 .../ernie/db/relaydescs/ArchiveWriter.java         |   12 ++++++++++--
 .../ernie/db/torperf/TorperfDownloader.java        |   10 ++++++++--
 6 files changed, 47 insertions(+), 22 deletions(-)

diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index 10d2baa..cb08df1 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -43,7 +43,16 @@ import org.torproject.ernie.db.main.RsyncDataProvider;
  * by the bridge to advertise their capabilities), and extra-info
  * descriptors (published by the bridge, mainly for statistical analysis).
  */
-public class SanitizedBridgesWriter {
+public class SanitizedBridgesWriter extends Thread {
+
+  private Configuration config;
+
+  /**
+   * Initializes this class.
+   */
+  public SanitizedBridgesWriter(Configuration config) {
+    this.config = config;
+  }
 
   /**
    * Logger for this class.
@@ -71,12 +80,7 @@ public class SanitizedBridgesWriter {
 
   private SecureRandom secureRandom;
 
-  /**
-   * Initializes this class.
-   */
-  public SanitizedBridgesWriter(Configuration config,
-      File statsDirectory) {
-
+  public void run() {
     File bridgeDirectoriesDirectory =
         new File(config.getBridgeSnapshotsDirectory());
     File sanitizedBridgesDirectory =
@@ -85,6 +89,7 @@ public class SanitizedBridgesWriter {
         config.getReplaceIPAddressesWithHashes();
     long limitBridgeSanitizingInterval =
         config.getLimitBridgeDescriptorMappings();
+    File statsDirectory = new File("stats");
 
     if (bridgeDirectoriesDirectory == null ||
         sanitizedBridgesDirectory == null || statsDirectory == null) {
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
index 3893495..d822d11 100644
--- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -29,9 +29,15 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.torproject.ernie.db.main.Configuration;
 import org.torproject.ernie.db.main.RsyncDataProvider;
 
-public class BridgePoolAssignmentsProcessor {
+public class BridgePoolAssignmentsProcessor extends Thread {
+
+  private Configuration config;
 
   public BridgePoolAssignmentsProcessor(Configuration config) {
+    this.config = config;
+  }
+
+  public void run() {
     File assignmentsDirectory =
         new File(config.getAssignmentsDirectory());
     File sanitizedAssignmentsDirectory =
diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
index f6f2865..bf67bb7 100644
--- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
+++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
@@ -21,8 +21,12 @@ import java.util.logging.Logger;
 import org.torproject.ernie.db.main.Configuration;
 import org.torproject.ernie.db.main.RsyncDataProvider;
 
-public class ExitListDownloader {
+public class ExitListDownloader extends Thread {
+
   public ExitListDownloader(Configuration config) {
+  }
+
+  public void run() {
     Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
     try {
       logger.fine("Downloading exit list...");
diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java
index 093f002..88f2372 100644
--- a/src/org/torproject/ernie/db/main/Main.java
+++ b/src/org/torproject/ernie/db/main/Main.java
@@ -2,7 +2,6 @@
  * See LICENSE for licensing information */
 package org.torproject.ernie.db.main;
 
-import java.io.File;
 import java.util.logging.Logger;
 
 import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter;
@@ -35,33 +34,30 @@ public class Main {
       System.exit(1);
     }
 
-    // Define stats directory for temporary files
-    File statsDirectory = new File("stats");
-
     // Import/download relay descriptors from the various sources
     if (config.getWriteDirectoryArchives()) {
-      new ArchiveWriter(config, statsDirectory);
+      new ArchiveWriter(config).start();
     }
 
     // Sanitize bridge descriptors
     if (config.getImportBridgeSnapshots() &&
         config.getWriteSanitizedBridges()) {
-      new SanitizedBridgesWriter(config, statsDirectory);
+      new SanitizedBridgesWriter(config).start();
     }
 
     // Download exit list and store it to disk
     if (config.getDownloadExitList()) {
-      new ExitListDownloader(config);
+      new ExitListDownloader(config).start();
     }
 
     // Process bridge pool assignments
     if (config.getProcessBridgePoolAssignments()) {
-      new BridgePoolAssignmentsProcessor(config);
+      new BridgePoolAssignmentsProcessor(config).start();
     }
 
     // Process Torperf files
     if (config.getProcessTorperfFiles()) {
-      new TorperfDownloader(config);
+      new TorperfDownloader(config).start();
     }
 
     // Remove lock file
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index a9cb604..5c3d2e9 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -27,17 +27,25 @@ import org.torproject.descriptor.impl.DescriptorParseException;
 import org.torproject.ernie.db.main.Configuration;
 import org.torproject.ernie.db.main.RsyncDataProvider;
 
-public class ArchiveWriter {
+public class ArchiveWriter extends Thread {
+
+  private Configuration config;
+
+  public ArchiveWriter(Configuration config) {
+    this.config = config;
+  }
+
   private Logger logger;
   private File outputDirectory;
   private DescriptorParser descriptorParser;
   private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0,
       storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
 
-  public ArchiveWriter(Configuration config, File statsDirectory) {
+  public void run() {
 
     File outputDirectory =
         new File(config.getDirectoryArchivesOutputDirectory());
+    File statsDirectory = new File("stats");
 
     this.logger = Logger.getLogger(ArchiveWriter.class.getName());
     this.outputDirectory = outputDirectory;
diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
index 5f09038..ef67ae7 100644
--- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
+++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
@@ -26,7 +26,13 @@ import org.torproject.ernie.db.main.RsyncDataProvider;
 /* Download possibly truncated Torperf .data and .extradata files from
  * configured sources, append them to the files we already have, and merge
  * the two files into the .tpf format. */
-public class TorperfDownloader {
+public class TorperfDownloader extends Thread {
+
+  private Configuration config;
+
+  public TorperfDownloader(Configuration config) {
+    this.config = config;
+  }
 
   private File torperfOutputDirectory = null;
   private SortedMap<String, String> torperfSources = null;
@@ -34,7 +40,7 @@ public class TorperfDownloader {
   private Logger logger = null;
   private SimpleDateFormat dateFormat;
 
-  public TorperfDownloader(Configuration config) {
+  public void run() {
 
     File torperfOutputDirectory =
         new File(config.getTorperfOutputDirectory());