[or-cvs] [ernie/master] Turned off autocommit when importing relay descriptors

[karsten at torproject.org]

Author: Kevin Berry <xckjb88 at gmail.com>
Date: Wed, 19 May 2010 17:37:22 -0400
Subject: Turned off autocommit when importing relay descriptors
Commit: 3d4757b1fedf0fbdd9e0b82ea2d3d607f2248a34

---
 src/RelayDescriptorDatabaseImporter.java |   39 +++++++++++++++++++++++++++--
 1 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/src/RelayDescriptorDatabaseImporter.java b/src/RelayDescriptorDatabaseImporter.java
index b3b311f..a7bc529 100644
--- a/src/RelayDescriptorDatabaseImporter.java
+++ b/src/RelayDescriptorDatabaseImporter.java
@@ -5,9 +5,21 @@ import java.util.logging.*;
 /**
  * Parse directory data.
  */
+
 public final class RelayDescriptorDatabaseImporter {
 
   /**
+   * How many records to commit with each database transaction.
+   */
+  private final long autoCommitCount = 500;
+
+  /**
+   * Keep track of the number of records committed before each transaction
+   */
+  private int rdsCount = 0;
+  private int rrsCount = 0;
+
+  /**
    * Relay descriptor database connection.
    */
   private Connection conn;
@@ -54,6 +66,9 @@ public final class RelayDescriptorDatabaseImporter {
       /* Connect to database. */
       this.conn = DriverManager.getConnection(connectionURL);
 
+      /* Turn autocommit off */
+      this.conn.setAutoCommit(false);
+
       /* Prepare statements. */
       this.psRs = conn.prepareStatement("SELECT COUNT(*) "
           + "FROM statusentry WHERE validafter = ? AND descriptor = ?");
@@ -110,7 +125,13 @@ public final class RelayDescriptorDatabaseImporter {
       this.psR.setBoolean(14, flags.contains("Valid"));
       this.psR.setBoolean(15, flags.contains("V2Dir"));
       this.psR.setBoolean(16, flags.contains("V3Dir"));
-      this.psR.execute();
+      this.psR.executeUpdate();
+      rrsCount++;
+      if (rrsCount % autoCommitCount == 0)  {
+        this.conn.commit();
+        rrsCount = 0;
+      }
+
     } catch (SQLException e) {
       this.logger.log(Level.WARNING, "Could not add network status "
           + "consensus entry.", e);
@@ -146,7 +167,13 @@ public final class RelayDescriptorDatabaseImporter {
       this.psD.setString(8, platform);
       this.psD.setTimestamp(9, new Timestamp(published), cal);
       this.psD.setLong(10, uptime);
-      this.psD.execute();
+      this.psD.executeUpdate();
+      rdsCount++;
+      if (rdsCount % autoCommitCount == 0)  {
+        this.conn.commit();
+        rdsCount = 0;
+      }
+
     } catch (SQLException e) {
       this.logger.log(Level.WARNING, "Could not add server descriptor.",
           e);
@@ -157,6 +184,13 @@ public final class RelayDescriptorDatabaseImporter {
    * Close the relay descriptor database connection.
    */
   public void closeConnection() {
+    /* commit any stragglers before closing */
+    try {
+      this.conn.commit();
+    }
+    catch (SQLException e)  {
+      this.logger.log(Level.WARNING, "Could not commit final records to database", e);
+    }
     try {
       this.conn.close();
     } catch (SQLException e) {
@@ -165,4 +199,3 @@ public final class RelayDescriptorDatabaseImporter {
     }
   }
 }
-
-- 
1.6.5