[or-cvs] r9937: Initial version of memory pool logic. Needs unit tests. Made (in tor/trunk: . doc src/common)
nickm at seul.org
nickm at seul.org
Wed Apr 11 00:30:23 UTC 2007
Author: nickm
Date: 2007-04-10 20:30:22 -0400 (Tue, 10 Apr 2007)
New Revision: 9937
Added:
tor/trunk/src/common/mempool.c
tor/trunk/src/common/mempool.h
Modified:
tor/trunk/
tor/trunk/doc/TODO
tor/trunk/src/common/Makefile.am
Log:
r12335 at catbus: nickm | 2007-04-10 16:53:48 -0400
Initial version of memory pool logic. Needs unit tests. Made to be easily separable from Tor.
Property changes on: tor/trunk
___________________________________________________________________
svk:merge ticket from /tor/branches/mempool [r12335] on 8246c3cf-6607-4228-993b-4d95d33730f1
Modified: tor/trunk/doc/TODO
===================================================================
--- tor/trunk/doc/TODO 2007-04-10 16:24:50 UTC (rev 9936)
+++ tor/trunk/doc/TODO 2007-04-11 00:30:22 UTC (rev 9937)
@@ -267,7 +267,8 @@
entries into a url. since + is a valid base64 char, only allow one
at a time. spec and then do.
- When we export something from foo.c file for testing purposes only,
- make a foo_test.h file for test.c to include.
+ make a foo_test.h file for test.c to include... or put them behind an
+ #ifdef FOO_PRIVATE.
- The Debian package now uses --verify-config when (re)starting,
to distinguish configuration errors from other errors. Perhaps
the RPM and other startup scripts should too?
Modified: tor/trunk/src/common/Makefile.am
===================================================================
--- tor/trunk/src/common/Makefile.am 2007-04-10 16:24:50 UTC (rev 9936)
+++ tor/trunk/src/common/Makefile.am 2007-04-11 00:30:22 UTC (rev 9937)
@@ -3,7 +3,7 @@
#CFLAGS = -Wall -Wpointer-arith -O2
-libor_a_SOURCES = log.c util.c compat.c container.c
+libor_a_SOURCES = log.c util.c compat.c container.c mempool.c
libor_crypto_a_SOURCES = crypto.c aes.c tortls.c torgzip.c
-noinst_HEADERS = log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h
+noinst_HEADERS = log.h crypto.h test.h util.h compat.h aes.h torint.h tortls.h strlcpy.c strlcat.c torgzip.h container.h ht.h mempool.h
Added: tor/trunk/src/common/mempool.c
===================================================================
--- tor/trunk/src/common/mempool.c 2007-04-10 16:24:50 UTC (rev 9936)
+++ tor/trunk/src/common/mempool.c 2007-04-11 00:30:22 UTC (rev 9937)
@@ -0,0 +1,293 @@
+/* Copyright 2007 Nick Mathewson */
+/* See LICENSE for licensing information */
+/* $Id: /tor/trunk/src/common/util.c 12153 2007-03-12T03:11:12.797278Z nickm $ */
+#include <stdlib.h>
+#include <string.h>
+
+#define MEMPOOL_PRIVATE
+#include "mempool.h"
+
+/* DRAWBACKS:
+ * - Not even slightly threadsafe.
+ * - Likes to have lots of items per chunks.
+ * - One pointer overhead per allocated thing. (The alternative is
+ * something like glib's use of an RB-tree to keep track of what
+ * chunk any given piece of memory is in.)
+ * - Only aligns allocated things to void* level: redefign ALIGNMENT_TYPE
+ * if you need doubles.
+ * - Could probably be optimized a bit; the representation contains
+ * a bit more info than it really needs to have.
+ */
+
+/* NOTES:
+ * - The algorithm is similar to the one used by Python, but assumes that
+ * we'll know in advance which objects we want to pool, and doesn't
+ * try to handle a zillion objects of weird different sizes.
+ */
+
+#if 1
+/* Tor dependencies */
+#include "orconfig.h"
+#include "util.h"
+#include "compat.h"
+#include "log.h"
+#define ALLOC(x) tor_malloc(x)
+#define FREE(x) tor_free(x)
+#define ASSERT(x) tor_assert(x)
+/* End Tor dependencies */
+#else
+#include <assert.h>
+#define PREDICT_UNLIKELY(x) (x)
+#define PREDICT_LIKELY(x) (x)
+#define ALLOC(x) malloc(x)
+#define FREE(x) free(x)
+#define STRUCT_OFFSET(tp, member) \
+ ((off_t) (((char*)&((tp*)0)->member)-(char*)0))
+#define ASSERT(x) assert(x)
+#endif
+
+/* Tuning parameters */
+/** DOCDOC */
+#define ALIGNMENT_TYPE void *
+/** DOCDOC */
+#define ALIGNMENT sizeof(void*)
+/** DOCDOC */
+#define MAX_CHUNK (8*(1L<<20))
+/** DOCDOC */
+#define MIN_CHUNK 4096
+
+/** DOCDOC */
+struct mp_allocated_t {
+ mp_chunk_t *in_chunk;
+ union {
+ mp_allocated_t *next_free;
+ char mem[1];
+ ALIGNMENT_TYPE _dummy;
+ };
+};
+
+/** DOCDOC */
+#define MP_CHUNK_MAGIC 0x09870123
+
+/** DOCDOC */
+#define CHUNK_OVERHEAD (sizeof(mp_chunk_t)-1)
+
+/** DOCDOC */
+#define A2M(a) (&(a)->mem)
+/** DOCDOC */
+#define M2A(p) ( ((char*)p) - STRUCT_OFFSET(mp_chunk_t, mem) )
+
+/* INVARIANT: every chunk can hold 2 or more items. */
+
+/** DOCDOC */
+static mp_chunk_t *
+mp_chunk_new(mp_pool_t *pool)
+{
+ size_t sz = pool->new_chunk_capacity * pool->item_alloc_size;
+ mp_chunk_t *chunk = ALLOC(CHUNK_OVERHEAD + sz);
+ memset(chunk, 0, sizeof(mp_chunk_t)); /* Doesn't clear the whole thing. */
+ chunk->magic = MP_CHUNK_MAGIC;
+ chunk->capacity = pool->new_chunk_capacity;
+ chunk->mem_size = sz;
+ chunk->next_mem = chunk->mem;
+ chunk->pool = pool;
+ return chunk;
+}
+
+/** DOCDOC */
+void *
+mp_pool_get(mp_pool_t *pool)
+{
+ mp_chunk_t *chunk;
+ mp_allocated_t *allocated;
+ if (PREDICT_LIKELY(pool->used_chunks != NULL)) {
+ chunk = pool->used_chunks;
+ } else if (pool->empty_chunks) {
+ /* Put the most recently emptied chunk on the used list. */
+ chunk = pool->empty_chunks;
+ pool->empty_chunks = chunk->next;
+ if (chunk->next)
+ chunk->next->prev = NULL;
+ chunk->next = pool->used_chunks;
+ if (chunk->next)
+ chunk->next->prev = chunk;
+ pool->used_chunks = chunk;
+ ASSERT(!chunk->prev);
+ } else {
+ /* Allocate a new chunk and add it to the used list. */
+ chunk = mp_chunk_new(pool);
+ chunk->next = pool->used_chunks;
+ if (chunk->next)
+ chunk->next->prev = chunk;
+ pool->used_chunks = chunk;
+ ASSERT(!chunk->prev);
+ }
+
+ ASSERT(chunk->n_allocated < chunk->capacity);
+
+ if (chunk->first_free) {
+ allocated = chunk->first_free;
+ chunk->first_free = allocated->next_free;
+ allocated->next_free = NULL; /* debugging */
+ } else {
+ ASSERT(chunk->next_mem + pool->item_alloc_size <
+ chunk->mem + chunk->mem_size);
+ allocated = (void*)chunk->next_mem;
+ chunk->next_mem += pool->item_alloc_size;
+ allocated->in_chunk = chunk;
+ }
+
+ ++chunk->n_allocated;
+ if (PREDICT_UNLIKELY(chunk->n_allocated == chunk->capacity)) {
+ /* This is now a full chunk. */
+ ASSERT(chunk == pool->used_chunks);
+ ASSERT(chunk->prev == NULL);
+ pool->used_chunks = chunk->next;
+ if (chunk->next)
+ chunk->next->prev = NULL;
+
+ chunk->next = pool->full_chunks;
+ pool->full_chunks->prev = chunk;
+ pool->full_chunks = chunk;
+ }
+
+ return A2M(allocated);
+}
+
+/** DOCDOC */
+void
+mp_pool_release(void *_item)
+{
+ mp_allocated_t *allocated = (void*) M2A(_item);
+ mp_chunk_t *chunk = allocated->in_chunk;
+
+ ASSERT(chunk);
+ ASSERT(chunk->magic == MP_CHUNK_MAGIC);
+ ASSERT(chunk->n_allocated > 0);
+
+ allocated->next_free = chunk->first_free;
+ chunk->first_free = allocated;
+
+ if (PREDICT_UNLIKELY(chunk->n_allocated == chunk->capacity)) {
+ /* This chunk was full and is about to be used. */
+ mp_pool_t *pool = chunk->pool;
+ /* unlink from full */
+ if (chunk->prev)
+ chunk->prev->next = chunk->next;
+ if (chunk->next)
+ chunk->next->prev = chunk->prev;
+ if (chunk == pool->full_chunks)
+ pool->full_chunks = chunk->next;
+
+ /* link to used */
+ chunk->next = pool->used_chunks;
+ chunk->prev = NULL;
+ if (chunk->next)
+ chunk->next->prev = chunk;
+ pool->used_chunks = chunk;
+ } else if (PREDICT_UNLIKELY(chunk->n_allocated == 1)) {
+ /* This was used and is about to be empty. */
+ mp_pool_t *pool = chunk->pool;
+ /* unlink from used */
+ if (chunk->prev)
+ chunk->prev->next = chunk->next;
+ if (chunk->next)
+ chunk->next->prev = chunk->prev;
+ if (chunk == pool->used_chunks)
+ pool->used_chunks = chunk->next;
+
+ /* link to empty */
+ chunk->next = pool->empty_chunks;
+ chunk->prev = NULL;
+ if (chunk->next)
+ chunk->next->prev = chunk;
+ pool->empty_chunks = chunk;
+
+ /* reset guts to defragment this chunk. */
+ chunk->first_free = NULL;
+ chunk->next_mem = chunk->mem;
+
+ ++pool->n_empty_chunks;
+ }
+ --chunk->n_allocated;
+}
+
+/** DOCDOC */
+mp_pool_t *
+mp_pool_new(size_t item_size, size_t chunk_capacity)
+{
+ mp_pool_t *pool;
+ size_t alloc_size;
+
+ pool = ALLOC(sizeof(mp_pool_t));
+ memset(pool, 0, sizeof(mp_pool_t));
+
+ /* First, minimal size with overhead. */
+ alloc_size = STRUCT_OFFSET(mp_allocated_t, mem) + item_size;
+ if (alloc_size < sizeof(mp_allocated_t))
+ alloc_size = sizeof(mp_allocated_t);
+
+ /* Then, round up to alignment. */
+ if (alloc_size % ALIGNMENT) {
+ alloc_size = alloc_size + ALIGNMENT - (alloc_size % ALIGNMENT);
+ }
+ if (alloc_size < ALIGNMENT)
+ alloc_size = ALIGNMENT;
+
+ ASSERT((alloc_size % ALIGNMENT) == 0);
+
+ if (chunk_capacity > MAX_CHUNK)
+ chunk_capacity = MAX_CHUNK;
+
+ if (chunk_capacity < alloc_size * 2 + CHUNK_OVERHEAD)
+ chunk_capacity = alloc_size * 2 + CHUNK_OVERHEAD;
+
+ if (chunk_capacity < MIN_CHUNK) /* Guess system page size. */
+ chunk_capacity = MIN_CHUNK;
+
+ pool->new_chunk_capacity = (chunk_capacity-CHUNK_OVERHEAD / alloc_size);
+ pool->item_alloc_size = alloc_size;
+
+ return pool;
+}
+
+/** DOCDOC */
+void
+mp_pool_clean(mp_pool_t *pool)
+{
+ if (pool->empty_chunks) {
+ mp_chunk_t *next, *chunk = pool->empty_chunks->next;
+ while (chunk) {
+ next = chunk->next;
+ FREE(chunk);
+ chunk = next;
+ }
+ pool->empty_chunks->next = NULL;
+ pool->n_empty_chunks = 1;
+ }
+}
+
+/** DOCDOC */
+static void
+destroy_chunks(mp_chunk_t *chunk)
+{
+ mp_chunk_t *next;
+ while (chunk) {
+ chunk->magic = 0xd3adb33f;
+ next = chunk->next;
+ FREE(chunk);
+ chunk = next;
+ }
+}
+
+/** DOCDOC */
+void
+mp_pool_destroy(mp_pool_t *pool)
+{
+ destroy_chunks(pool->empty_chunks);
+ destroy_chunks(pool->used_chunks);
+ destroy_chunks(pool->full_chunks);
+ memset(pool, 0xe0, sizeof(mp_pool_t));
+ FREE(pool);
+}
+
Added: tor/trunk/src/common/mempool.h
===================================================================
--- tor/trunk/src/common/mempool.h 2007-04-10 16:24:50 UTC (rev 9936)
+++ tor/trunk/src/common/mempool.h 2007-04-11 00:30:22 UTC (rev 9937)
@@ -0,0 +1,51 @@
+/* Copyright 2007 Nick Mathewson */
+/* See LICENSE for licensing information */
+/* $Id: /tor/trunk/src/common/util.c 12153 2007-03-12T03:11:12.797278Z nickm $ */
+
+/**
+ * \file util.h
+ * \brief Headers for mempool.c
+ **/
+
+#ifndef MEMPOOL_H
+#define MEMPOOL_H
+
+typedef struct mp_pool_t mp_pool_t;
+
+void *mp_pool_get(mp_pool_t *pool);
+void mp_pool_release(void *item);
+mp_pool_t *mp_pool_new(size_t item_size, unsigned int n_per_chunk);
+void mp_pool_clean(mp_pool_t *pool);
+void mp_pool_destroy(mp_pool_t *pool);
+
+#ifdef MEMPOOL_PRIVATE
+typedef struct mp_allocated_t mp_allocated_t;
+typedef struct mp_chunk_t mp_chunk_t;
+
+/** DOCDOC */
+struct mp_chunk_t {
+ unsigned long magic;
+ mp_chunk_t *next;
+ mp_chunk_t *prev;
+ mp_pool_t *pool;
+ mp_allocated_t *first_free;
+ int n_allocated;
+ int capacity;
+ size_t mem_size;
+ char *next_mem;
+ char mem[1];
+};
+
+/** DOCDOC */
+struct mp_pool_t {
+ mp_chunk_t *empty_chunks;
+ mp_chunk_t *used_chunks;
+ mp_chunk_t *full_chunks;
+ int n_empty_chunks;
+ size_t new_chunk_capacity;
+ size_t item_alloc_size;
+};
+#endif
+
+#endif
+
More information about the tor-commits
mailing list