[tor-commits] [collector/master] Add support for Bifroest's bridge descriptor tarballs.

[karsten at torproject.org]

commit 46ab6609854e1e2f0c3d55a52f9ada1c2dd59793
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Fri Aug 26 12:23:22 2016 +0200

    Add support for Bifroest's bridge descriptor tarballs.
    
    Implements #20037.
---
 CHANGELOG.md                                       |  3 ++
 .../bridgedescs/BridgeDescriptorParser.java        |  5 +--
 .../bridgedescs/BridgeSnapshotReader.java          | 36 +++++++++++++++++++---
 .../bridgedescs/SanitizedBridgesWriter.java        |  6 ++--
 4 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a8be9e2..cf1ea12 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changes in version 1.1.0 - 2016-08-31
 
+ * Medium changes
+   - Add support for Bifroest's bridge descriptor tarballs.
+
  * Minor changes
    - Remove quotes around base URL in index.json.
 
diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java
index 511b171..d40052e 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java
@@ -27,7 +27,7 @@ public class BridgeDescriptorParser {
 
   /** Parses the first line of the given descriptor data to determine the
    * descriptor type and passes it to the sanitized bridges writer. */
-  public void parse(byte[] allData, String dateTime)
+  public void parse(byte[] allData, String dateTime, String authorityFingerprint)
       throws ConfigurationException {
     try {
       BufferedReader br = new BufferedReader(new StringReader(
@@ -45,7 +45,8 @@ public class BridgeDescriptorParser {
         }
       } else {
         if (this.sbw != null) {
-          this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);
+          this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime,
+              authorityFingerprint);
         }
       }
     } catch (IOException e) {
diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java
index 25bb2ea..046db7d 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java
@@ -100,9 +100,35 @@ public class BridgeSnapshotReader {
               }
               BufferedInputStream bis = new BufferedInputStream(tais);
               String fn = pop.getName();
-              String dateTime = fn.substring(11, 21) + " "
-                    + fn.substring(22, 24) + ":" + fn.substring(24, 26)
-                    + ":" + fn.substring(26, 28);
+              String[] fnParts = fn.split("-");
+              if (fnParts.length != 5) {
+                logger.warn("Invalid bridge descriptor tarball file name: "
+                    + fn + ".  Skipping.");
+                continue;
+              }
+              String authorityPart = String.format("%s-%s-", fnParts[0],
+                  fnParts[1]);
+              String datePart = String.format("%s-%s-%s", fnParts[2],
+                  fnParts[3], fnParts[4]);
+              String authorityFingerprint;
+              switch (authorityPart) {
+                case "from-tonga-":
+                  authorityFingerprint =
+                      "4A0CCD2DDC7995083D73F5D667100C8A5831F16D";
+                  break;
+                case "from-bifroest-":
+                  authorityFingerprint =
+                      "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1";
+                  break;
+                default:
+                  logger.warn("Did not recognize the bridge authority that "
+                      + "generated " + fn + ".  Skipping.");
+                  continue;
+              }
+              String dateTime = datePart.substring(0, 10) + " "
+                  + datePart.substring(11, 13) + ":"
+                  + datePart.substring(13, 15) + ":"
+                  + datePart.substring(15, 17);
               while ((tais.getNextTarEntry()) != null) {
                 ByteArrayOutputStream baos = new ByteArrayOutputStream();
                 int len;
@@ -130,7 +156,7 @@ public class BridgeSnapshotReader {
                 if (firstLine.startsWith("published ")
                     || firstLine.startsWith("flag-thresholds ")
                     || firstLine.startsWith("r ")) {
-                  bdp.parse(allData, dateTime);
+                  bdp.parse(allData, dateTime, authorityFingerprint);
                   parsedStatuses++;
                 } else if (descriptorImportHistory.contains(fileDigest)) {
                   /* Skip server descriptors or extra-info descriptors if
@@ -167,7 +193,7 @@ public class BridgeSnapshotReader {
                         DigestUtils.sha(descBytes));
                     if (!descriptorImportHistory.contains(
                         descriptorDigest)) {
-                      bdp.parse(descBytes, dateTime);
+                      bdp.parse(descBytes, dateTime, authorityFingerprint);
                       descriptorImportHistory.add(descriptorDigest);
                       if (firstLine.startsWith("router ")) {
                         parsedServerDescriptors++;
diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index dcac024..b787f78 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -379,7 +379,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
    * Sanitizes a network status and writes it to disk.
    */
   public void sanitizeAndStoreNetworkStatus(byte[] data,
-      String publicationTime) throws ConfigurationException {
+      String publicationTime, String authorityFingerprint)
+      throws ConfigurationException {
 
     if (this.persistenceProblemWithSecrets) {
       /* There's a persistence problem, so we shouldn't scrub more IP
@@ -546,8 +547,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
       File tarballFile = new File(
           this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear
           + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth
-          + sday + "-" + stime + "-"
-          + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D");
+          + sday + "-" + stime + "-" + authorityFingerprint);
       File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
           "bridge-descriptors/statuses/" + tarballFile.getName());
       File[] outputFiles = new File[] { tarballFile, rsyncFile };