[tor-commits] [metrics-lib/master] Make queue size of descriptor reader configurable.

[karsten at torproject.org]

commit 8722de70447434ebb0d92551c8622566efb6d800
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date:   Tue May 27 20:53:04 2014 +0200

    Make queue size of descriptor reader configurable.
    
    By default, the descriptor reader puts up to 100 parsed descriptor files
    in a queue in order to hand them out as quickly as possible.  But if
    descriptor files contain hundreds or even thousands of descriptors, that
    default may be too high.  Add a new method to make it configurable.
---
 src/org/torproject/descriptor/DescriptorReader.java      |    5 +++++
 .../torproject/descriptor/impl/BlockingIteratorImpl.java |   11 +++++++++--
 .../torproject/descriptor/impl/DescriptorReaderImpl.java |   14 +++++++++++++-
 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/org/torproject/descriptor/DescriptorReader.java b/src/org/torproject/descriptor/DescriptorReader.java
index 2783bd4..5fe8197 100644
--- a/src/org/torproject/descriptor/DescriptorReader.java
+++ b/src/org/torproject/descriptor/DescriptorReader.java
@@ -49,6 +49,11 @@ public interface DescriptorReader {
    * explicitly. */
   public void setFailUnrecognizedDescriptorLines();
 
+  /* Don't keep more than this number of parsed descriptor files in the
+   * queue.  The default is 100, but if descriptor files contain hundreds
+   * or even thousands of descriptors, that default may be too high. */
+  public void setMaxDescriptorFilesInQueue(int max);
+
   /* Read the previously configured descriptors and make them available
    * via the returned blocking iterator.  Whenever the reader runs out of
    * descriptors and expects to provide more shortly after, it blocks the
diff --git a/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java b/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java
index e23034a..477e6f1 100644
--- a/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java
+++ b/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java
@@ -15,19 +15,26 @@ public class BlockingIteratorImpl<T> implements Iterator<T> {
   /* Queue containing produced elemnts waiting for consumers. */
   private Queue<T> queue = new LinkedList<T>();
 
+  /* Maximum number of elements in queue. */
+  private int maxQueueSize = 100;
+
   /* Restrict object construction to the impl package. */
   protected BlockingIteratorImpl() {
   }
 
+  /* Create instance with maximum queue size. */
+  protected BlockingIteratorImpl(int maxQueueSize) {
+    this.maxQueueSize = maxQueueSize;
+  }
+
   /* Add an object to the queue if there's still room. */
-  final int MAX_DESCRIPTORS = 100;
   protected synchronized void add(T object) {
     if (this.outOfDescriptors) {
       throw new IllegalStateException("Internal error: Adding results to "
           + "descriptor queue not allowed after sending end-of-stream "
           + "object.");
     }
-    while (this.queue.size() >= this.MAX_DESCRIPTORS) {
+    while (this.queue.size() >= this.maxQueueSize) {
       try {
         wait();
       } catch (InterruptedException e) {
diff --git a/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java
index e3fb908..d679154 100644
--- a/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java
+++ b/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java
@@ -93,6 +93,15 @@ public class DescriptorReaderImpl implements DescriptorReader {
     this.failUnrecognizedDescriptorLines = true;
   }
 
+  private Integer maxDescriptorFilesInQueue = null;
+  public void setMaxDescriptorFilesInQueue(int max) {
+    if (this.hasStartedReading) {
+      throw new IllegalStateException("Reconfiguration is not permitted "
+          + "after starting to read.");
+    }
+    this.maxDescriptorFilesInQueue = max;
+  }
+
   private DescriptorReaderRunnable reader;
   public Iterator<DescriptorFile> readDescriptors() {
     if (this.hasStartedReading) {
@@ -101,7 +110,10 @@ public class DescriptorReaderImpl implements DescriptorReader {
     }
     this.hasStartedReading = true;
     BlockingIteratorImpl<DescriptorFile> descriptorQueue =
-        new BlockingIteratorImpl<DescriptorFile>();
+        this.maxDescriptorFilesInQueue == null
+        ? new BlockingIteratorImpl<DescriptorFile>()
+        : new BlockingIteratorImpl<DescriptorFile>(
+        this.maxDescriptorFilesInQueue);
     this.reader = new DescriptorReaderRunnable(this.directories,
         this.tarballs, descriptorQueue, this.historyFile,
         this.excludedFiles, this.failUnrecognizedDescriptorLines);