[tor-commits] [onionoo/master] Avoid parsing server descriptors for weights histories.
karsten at torproject.org
karsten at torproject.org
Fri Jul 12 19:10:00 UTC 2013
commit 2f305253463c3feb42aad4e8477f4aac199b026e
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Jul 11 14:18:06 2013 +0200
Avoid parsing server descriptors for weights histories.
Instead, store advertised bandwidths in weights status files. Prepares
using a parse history for server descriptors.
---
src/org/torproject/onionoo/Main.java | 4 +
src/org/torproject/onionoo/WeightsDataWriter.java | 92 +++++++++++++++++----
2 files changed, 79 insertions(+), 17 deletions(-)
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index 9c02319..0d29ab5 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -92,6 +92,8 @@ public class Main {
printStatus("Updating weights data.");
wdw.setCurrentNodes(currentNodes);
printStatusTime("Set current node fingerprints");
+ wdw.updateWeightsHistories();
+ printStatusTime("Updated weights histories");
wdw.writeWeightsDataFiles();
printStatusTime("Wrote weights data files");
// TODO Evaluate overhead of not deleting obsolete weights files. An
@@ -99,6 +101,8 @@ public class Main {
// which allows us to run ndw and wdw in parallel in the future.
wdw.deleteObsoleteWeightsDataFiles();
printStatusTime("Deleted obsolete weights files");
+ wdw.updateAdvertisedBandwidths();
+ printStatusTime("Updated advertised bandwidths");
printStatus("Shutting down.");
dso.writeHistoryFiles();
diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java
index 260cdc8..ba4482b 100644
--- a/src/org/torproject/onionoo/WeightsDataWriter.java
+++ b/src/org/torproject/onionoo/WeightsDataWriter.java
@@ -8,10 +8,12 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Scanner;
+import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TimeZone;
@@ -54,21 +56,25 @@ public class WeightsDataWriter implements DescriptorListener {
}
}
- public void setCurrentNodes(
- SortedMap<String, NodeStatus> currentNodes) {
- this.currentFingerprints.addAll(currentNodes.keySet());
+ private Set<RelayNetworkStatusConsensus> consensuses =
+ new HashSet<RelayNetworkStatusConsensus>();
+
+ private void processRelayNetworkConsensus(
+ RelayNetworkStatusConsensus consensus) {
+ this.consensuses.add(consensus);
}
+ private Set<String> updateAdvertisedBandwidths =
+ new HashSet<String>();
+
+ private Map<String, Set<String>> descriptorDigestsByFingerprint =
+ new HashMap<String, Set<String>>();
+
private Map<String, Integer> advertisedBandwidths =
new HashMap<String, Integer>();
private void processRelayServerDescriptor(
ServerDescriptor serverDescriptor) {
- /* Read advertised bandwidths of all server descriptors in
- * in/relay-descriptors/server-descriptors/ to memory. Ideally, we'd
- * skip descriptors that we read before and obtain their advertised
- * bandwidths from some temp file. This approach should do for now,
- * though. */
String digest = serverDescriptor.getServerDescriptorDigest().
toUpperCase();
int advertisedBandwidth = Math.min(Math.min(
@@ -76,19 +82,34 @@ public class WeightsDataWriter implements DescriptorListener {
serverDescriptor.getBandwidthObserved()),
serverDescriptor.getBandwidthRate());
this.advertisedBandwidths.put(digest, advertisedBandwidth);
+ String fingerprint = serverDescriptor.getFingerprint();
+ this.updateAdvertisedBandwidths.add(fingerprint);
+ if (!this.descriptorDigestsByFingerprint.containsKey(
+ fingerprint)) {
+ this.descriptorDigestsByFingerprint.put(fingerprint,
+ new HashSet<String>());
+ }
+ this.descriptorDigestsByFingerprint.get(fingerprint).add(digest);
}
- private void processRelayNetworkConsensus(
- RelayNetworkStatusConsensus consensus) {
- long validAfterMillis = consensus.getValidAfterMillis(),
- freshUntilMillis = consensus.getFreshUntilMillis();
- SortedMap<String, double[]> pathSelectionWeights =
- this.calculatePathSelectionProbabilities(consensus);
- this.updateWeightsHistory(validAfterMillis, freshUntilMillis,
- pathSelectionWeights);
+ public void setCurrentNodes(
+ SortedMap<String, NodeStatus> currentNodes) {
+ this.currentFingerprints.addAll(currentNodes.keySet());
}
- private static final int HISTORY_UPDATER_WORKERS_NUM = 4;
+ public void updateWeightsHistories() {
+ for (RelayNetworkStatusConsensus consensus : this.consensuses) {
+ long validAfterMillis = consensus.getValidAfterMillis(),
+ freshUntilMillis = consensus.getFreshUntilMillis();
+ SortedMap<String, double[]> pathSelectionWeights =
+ this.calculatePathSelectionProbabilities(consensus);
+ this.updateWeightsHistory(validAfterMillis, freshUntilMillis,
+ pathSelectionWeights);
+ }
+ }
+
+ // TODO Use 4 workers once threading problems are solved.
+ private static final int HISTORY_UPDATER_WORKERS_NUM = 1;
private void updateWeightsHistory(long validAfterMillis,
long freshUntilMillis,
SortedMap<String, double[]> pathSelectionWeights) {
@@ -189,6 +210,10 @@ public class WeightsDataWriter implements DescriptorListener {
String serverDescriptorDigest = relay.getDescriptor().
toUpperCase();
double advertisedBandwidth = 0.0;
+ if (!this.advertisedBandwidths.containsKey(
+ serverDescriptorDigest)) {
+ this.readHistoryFromDisk(fingerprint);
+ }
if (this.advertisedBandwidths.containsKey(
serverDescriptorDigest)) {
advertisedBandwidth = (double) this.advertisedBandwidths.get(
@@ -279,6 +304,20 @@ public class WeightsDataWriter implements DescriptorListener {
while (s.hasNextLine()) {
String line = s.nextLine();
String[] parts = line.split(" ");
+ if (parts.length == 2) {
+ String descriptorDigest = parts[0];
+ int advertisedBandwidth = Integer.parseInt(parts[1]);
+ if (!this.descriptorDigestsByFingerprint.containsKey(
+ fingerprint)) {
+ this.descriptorDigestsByFingerprint.put(fingerprint,
+ new HashSet<String>());
+ }
+ this.descriptorDigestsByFingerprint.get(fingerprint).add(
+ descriptorDigest);
+ this.advertisedBandwidths.put(descriptorDigest,
+ advertisedBandwidth);
+ continue;
+ }
if (parts.length != 9) {
System.err.println("Illegal line '" + line + "' in weights "
+ "history for fingerprint '" + fingerprint + "'. "
@@ -372,6 +411,17 @@ public class WeightsDataWriter implements DescriptorListener {
private void writeHistoryToDisk(String fingerprint,
SortedMap<long[], double[]> history) {
StringBuilder sb = new StringBuilder();
+ if (this.descriptorDigestsByFingerprint.containsKey(fingerprint)) {
+ for (String descriptorDigest :
+ this.descriptorDigestsByFingerprint.get(fingerprint)) {
+ if (this.advertisedBandwidths.containsKey(descriptorDigest)) {
+ int advertisedBandwidth =
+ this.advertisedBandwidths.get(descriptorDigest);
+ sb.append(descriptorDigest + " "
+ + String.valueOf(advertisedBandwidth) + "\n");
+ }
+ }
+ }
SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss");
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -568,5 +618,13 @@ public class WeightsDataWriter implements DescriptorListener {
this.documentStore.remove(WeightsDocument.class, fingerprint);
}
}
+
+ public void updateAdvertisedBandwidths() {
+ for (String fingerprint : this.updateAdvertisedBandwidths) {
+ SortedMap<long[], double[]> history =
+ this.readHistoryFromDisk(fingerprint);
+ this.writeHistoryToDisk(fingerprint, history);
+ }
+ }
}
More information about the tor-commits
mailing list