[or-cvs] [metrics-db/master] Improve bandwidth history statistics.

[karsten at torproject.org]

Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed, 13 Oct 2010 12:31:31 +0200
Subject: Improve bandwidth history statistics.
Commit: 8e443e26110798f911e5fa814f022f54a989044c

Fix materialized view for daily written/read dir bytes.

Only add history lines ending less than 1 week before or after the descriptor
publication time. See task 1926 for details.
---
 db/tordir.sql                                      |   13 +++++++++----
 .../torproject/ernie/db/RelayDescriptorParser.java |   13 +++++++++++--
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/db/tordir.sql b/db/tordir.sql
index 56789cf..6be310b 100644
--- a/db/tordir.sql
+++ b/db/tordir.sql
@@ -499,7 +499,7 @@ CREATE OR REPLACE FUNCTION refresh_total_bandwidth() RETURNS INTEGER AS $$
     END;
 $$ LANGUAGE plpgsql;
 
--- FUNCTION refresh_network_size()
+-- FUNCTION refresh_total_bwhist()
 CREATE OR REPLACE FUNCTION refresh_total_bwhist() RETURNS INTEGER AS $$
   BEGIN
   DELETE FROM total_bwhist WHERE date IN (SELECT * FROM updates);
@@ -507,9 +507,14 @@ CREATE OR REPLACE FUNCTION refresh_total_bwhist() RETURNS INTEGER AS $$
   SELECT date,
          SUM(read) AS read,
          SUM(written) AS written,
-         SUM(CASE WHEN dirwritten IS NULL THEN NULL ELSE written END)
-             AS dirwritten,
-         SUM(CASE WHEN dirread IS NULL THEN NULL ELSE read END) AS dirread
+         SUM(dirread) * (SUM(written) + SUM(read)) / (1
+           + SUM(CASE WHEN dirwritten IS NULL THEN NULL ELSE written END)
+           + SUM(CASE WHEN dirread IS NULL THEN NULL ELSE read END))
+           AS dirread,
+         SUM(dirwritten) * (SUM(written) + SUM(read)) / (1
+           + SUM(CASE WHEN dirwritten IS NULL THEN NULL ELSE written END)
+           + SUM(CASE WHEN dirread IS NULL THEN NULL ELSE read END))
+           AS dirwritten
   FROM (
     SELECT fingerprint,
            DATE(intervalend) AS date,
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
index d82b041..1dc10c4 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java
@@ -370,8 +370,17 @@ public class RelayDescriptorParser {
             String[] parts = line.split(" ");
             if (parts.length == 6) {
               String type = parts[0];
-              long intervalEnd = dateTimeFormat.parse(parts[1] + " "
-                  + parts[2]).getTime();
+              String intervalEndTime = parts[1] + " " + parts[2];
+              long intervalEnd = dateTimeFormat.parse(intervalEndTime).
+                  getTime();
+              if (Math.abs(published - intervalEnd) >
+                  7L * 24L * 60L * 60L * 1000L) {
+                this.logger.fine("Extra-info descriptor publication time "
+                    + publishedTime + " and last interval time "
+                    + intervalEndTime + " in " + type + " line differ by "
+                    + "more than 7 days! Not adding this line!");
+                continue;
+              }
               try {
                 long intervalLength = Long.parseLong(parts[3].
                     substring(1));
-- 
1.7.1