[or-cvs] [metrics-db/master] Memorize dirreq-stats timestamps, not only dates.
karsten at torproject.org
karsten at torproject.org
Tue Nov 23 07:45:29 UTC 2010
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue, 23 Nov 2010 07:56:49 +0100
Subject: Memorize dirreq-stats timestamps, not only dates.
Commit: 58d86913f02f9f6c1abd529c129bf50945764508
---
db/tordir.sql | 6 +-
.../ernie/db/DirreqStatsFileHandler.java | 86 +++++++++++++-------
.../torproject/ernie/db/RelayDescriptorParser.java | 14 +++-
3 files changed, 70 insertions(+), 36 deletions(-)
diff --git a/db/tordir.sql b/db/tordir.sql
index 43064d6..7638498 100644
--- a/db/tordir.sql
+++ b/db/tordir.sql
@@ -561,10 +561,12 @@ CREATE TABLE bridge_network_size (
-- Contains daily users by country.
CREATE TABLE dirreq_stats (
source CHARACTER(40) NOT NULL,
- "date" DATE NOT NULL,
+ statsend TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+ seconds INTEGER NOT NULL,
country CHARACTER(2) NOT NULL,
requests INTEGER NOT NULL,
- CONSTRAINT dirreq_stats_pkey PRIMARY KEY (source, "date", country)
+ CONSTRAINT dirreq_stats_pkey
+ PRIMARY KEY (source, statsend, seconds, country)
);
-- TABLE bridge_stats
diff --git a/src/org/torproject/ernie/db/DirreqStatsFileHandler.java b/src/org/torproject/ernie/db/DirreqStatsFileHandler.java
index 21ae0f8..6fc9014 100644
--- a/src/org/torproject/ernie/db/DirreqStatsFileHandler.java
+++ b/src/org/torproject/ernie/db/DirreqStatsFileHandler.java
@@ -28,7 +28,8 @@ public class DirreqStatsFileHandler {
/**
* Directory requests by directory and date. Map keys are directory and
- * date written as "directory,date", map values are country-user maps.
+ * date written as "directory,statsend,seconds", map values are
+ * country-user maps.
*/
private SortedMap<String, Map<String, String>> dirreqs;
@@ -49,6 +50,9 @@ public class DirreqStatsFileHandler {
/* Database connection string. */
private String connectionURL = null;
+ /* Format for parsing dirreq-stats-end timestamps. */
+ private SimpleDateFormat dateTimeFormat = null;
+
/**
* Initializes this class, including reading in previous results from
* <code>stats/dirreq-stats</code>.
@@ -69,6 +73,10 @@ public class DirreqStatsFileHandler {
/* Initialize database connection string. */
this.connectionURL = connectionURL;
+ /* Initialize format to parse dirreq-stats-end lines. */
+ this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
/* Initialize logger. */
this.logger = Logger.getLogger(
DirreqStatsFileHandler.class.getName());
@@ -85,14 +93,14 @@ public class DirreqStatsFileHandler {
/* The first line should contain headers that we need to parse
* in order to learn what countries we were interested in when
* writing this file. */
- if (!line.startsWith("directory,date,")) {
+ if (!line.startsWith("directory,statsend,seconds,")) {
this.logger.warning("Incorrect first line '" + line + "' in "
+ this.dirreqStatsFile.getAbsolutePath() + "! This line "
+ "should contain headers! Aborting to read in this "
+ "file!");
} else {
String[] headers = line.split(",");
- for (int i = 2; i < headers.length; i++) {
+ for (int i = 3; i < headers.length; i++) {
if (!headers[i].equals("all")) {
this.countries.add(headers[i]);
}
@@ -109,9 +117,10 @@ public class DirreqStatsFileHandler {
break;
}
String directory = parts[0];
- String date = parts[1];
+ String statsEnd = parts[1];
+ long seconds = Long.parseLong(parts[2]);
Map<String, String> obs = new HashMap<String, String>();
- for (int i = 2; i < parts.length; i++) {
+ for (int i = 3; i < parts.length; i++) {
if (parts[i].equals("NA")) {
continue;
}
@@ -121,7 +130,7 @@ public class DirreqStatsFileHandler {
obs.put(headers[i], parts[i]);
}
}
- this.addObs(directory, date, obs);
+ this.addObs(directory, statsEnd, seconds, obs);
}
}
}
@@ -143,12 +152,12 @@ public class DirreqStatsFileHandler {
* Adds observations on the number of directory requests by country as
* seen on a directory at a given date.
*/
- public void addObs(String directory, String date,
+ public void addObs(String directory, String statsEnd, long seconds,
Map<String, String> obs) {
for (String country : obs.keySet()) {
this.countries.add(country);
}
- String key = directory + "," + date;
+ String key = directory + "," + statsEnd + "," + seconds;
if (!this.dirreqs.containsKey(key)) {
this.logger.finer("Adding new directory request numbers: " + key);
this.dirreqs.put(key, obs);
@@ -178,7 +187,7 @@ public class DirreqStatsFileHandler {
BufferedWriter bw = new BufferedWriter(new FileWriter(
this.dirreqStatsFile));
/* Write header. */
- bw.append("directory,date");
+ bw.append("directory,statsend,seconds");
for (String country : this.countries) {
if (country.equals("zy")) {
bw.append(",all");
@@ -224,13 +233,15 @@ public class DirreqStatsFileHandler {
for (Map.Entry<String, Map<String, String>> e :
this.dirreqs.entrySet()) {
String[] parts = e.getKey().split(",");
- String directory = parts[0];
- String date = parts[1];
+ String source = parts[0];
+ String statsEnd = parts[1];
+ String seconds = parts[2];
Map<String, String> obs = e.getValue();
int i = 0;
for (String country : this.countries) {
if (obs.containsKey(country)) {
- String key = directory + "," + date + "," + country;
+ String key = source + "," + statsEnd + "," + seconds + ","
+ + country;
String requests = "" + obs.get(country);
insertRows.put(key, requests);
}
@@ -240,12 +251,16 @@ public class DirreqStatsFileHandler {
conn.setAutoCommit(false);
Statement statement = conn.createStatement();
ResultSet rs = statement.executeQuery(
- "SELECT source, date, country, requests FROM dirreq_stats");
+ "SELECT source, statsend, seconds, country, requests "
+ + "FROM dirreq_stats");
while (rs.next()) {
String source = rs.getString(1);
- String date = rs.getDate(2).toString();
- String country = rs.getString(3);
- String key = source + "," + date + "," + country;
+ String statsEnd = this.dateTimeFormat.format(
+ rs.getTimestamp(2).getTime());
+ long seconds = rs.getLong(3);
+ String country = rs.getString(4);
+ String key = source + "," + statsEnd + "," + seconds + ","
+ + country;
if (insertRows.containsKey(key)) {
String insertRow = insertRows.remove(key);
long oldUsers = rs.getLong(4);
@@ -258,36 +273,42 @@ public class DirreqStatsFileHandler {
rs.close();
PreparedStatement psU = conn.prepareStatement(
"UPDATE dirreq_stats SET requests = ? "
- + "WHERE source = ? AND date = ? AND country = ?");
+ + "WHERE source = ? AND statsend = ? AND seconds = ? "
+ + "AND country = ?");
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
for (Map.Entry<String, String> e : updateRows.entrySet()) {
String[] keyParts = e.getKey().split(",");
- String valueParts = e.getValue();
String source = keyParts[0];
- java.sql.Date date = java.sql.Date.valueOf(keyParts[1]);
- String country = keyParts[2];
- long requests = Long.parseLong(valueParts);
+ Timestamp statsEnd = new Timestamp(this.dateTimeFormat.parse(
+ keyParts[1]).getTime());
+ long seconds = Long.parseLong(keyParts[2]);
+ String country = keyParts[3];
+ long requests = Long.parseLong(e.getValue());
psU.clearParameters();
psU.setLong(1, requests);
psU.setString(2, source);
- psU.setDate(3, date);
- psU.setString(4, country);
+ psU.setTimestamp(3, statsEnd, cal);
+ psU.setLong(4, seconds);
+ psU.setString(5, country);
psU.executeUpdate();
}
PreparedStatement psI = conn.prepareStatement(
- "INSERT INTO dirreq_stats (requests, source, date, "
- + "country) VALUES (?, ?, ?, ?)");
+ "INSERT INTO dirreq_stats (requests, source, statsend, "
+ + "seconds, country) VALUES (?, ?, ?, ?, ?)");
for (Map.Entry<String, String> e : insertRows.entrySet()) {
String[] keyParts = e.getKey().split(",");
- String valueParts = e.getValue();
String source = keyParts[0];
- java.sql.Date date = java.sql.Date.valueOf(keyParts[1]);
- String country = keyParts[2];
- long requests = Long.parseLong(valueParts);
+ Timestamp statsEnd = new Timestamp(this.dateTimeFormat.parse(
+ keyParts[1]).getTime());
+ long seconds = Long.parseLong(keyParts[2]);
+ String country = keyParts[3];
+ long requests = Long.parseLong(e.getValue());
psI.clearParameters();
psI.setLong(1, requests);
psI.setString(2, source);
- psI.setDate(3, date);
- psI.setString(4, country);
+ psI.setTimestamp(3, statsEnd, cal);
+ psI.setLong(4, seconds);
+ psI.setString(5, country);
psI.executeUpdate();
}
conn.commit();
@@ -295,6 +316,9 @@ public class DirreqStatsFileHandler {
} catch (SQLException e) {
logger.log(Level.WARNING, "Failed to add directory requests by "
+ "country to database.", e);
+ } catch (ParseException e) {
+ logger.log(Level.WARNING, "Failed to add directory requests by "
+ + "country to database.", e);
}
}
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
index 40e3b56..27226f3 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java
@@ -325,7 +325,8 @@ public class RelayDescriptorParser {
String publishedTime = null, relayIdentifier = line.split(" ")[2];
long published = -1L;
String dir = line.split(" ")[2];
- String date = null;
+ String statsEnd = null;
+ long seconds = -1L;
SortedMap<String, String> bandwidthHistory =
new TreeMap<String, String>();
boolean skip = false;
@@ -369,7 +370,14 @@ public class RelayDescriptorParser {
}
}
} else if (line.startsWith("dirreq-stats-end ")) {
- date = line.split(" ")[1];
+ String[] parts = line.split(" ");
+ if (parts.length < 5) {
+ this.logger.warning("Could not parse dirreq-stats-end "
+ + "line '" + line + "' in descriptor. Skipping.");
+ break;
+ }
+ statsEnd = parts[1] + " " + parts[2];
+ seconds = Long.parseLong(parts[3].substring(1));
} else if (line.startsWith("dirreq-v3-reqs ")
&& line.length() > "dirreq-v3-reqs ".length()) {
if (this.dsfh != null) {
@@ -385,7 +393,7 @@ public class RelayDescriptorParser {
obs.put(country, "" + users);
}
obs.put("zy", "" + allUsers);
- this.dsfh.addObs(dir, date, obs);
+ this.dsfh.addObs(dir, statsEnd, seconds, obs);
} catch (NumberFormatException e) {
this.logger.log(Level.WARNING, "Could not parse "
+ "dirreq-v3-reqs line '" + line + "' in descriptor. "
--
1.7.1
More information about the tor-commits
mailing list