[tor-commits] [sbws/master] Complete info about required disk space

[juga at torproject.org]

commit 1e916ea2452d499dacae9ac1e64385cd73f10f20
Author: juga0 <juga at riseup.net>
Date:   Thu Sep 6 16:55:51 2018 +0000

    Complete info about required disk space
---
 sbws/util/fs.py | 77 +++++++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 61 insertions(+), 16 deletions(-)

diff --git a/sbws/util/fs.py b/sbws/util/fs.py
index c7a5de7..0095f2a 100644
--- a/sbws/util/fs.py
+++ b/sbws/util/fs.py
@@ -5,37 +5,82 @@ import shutil
 
 log = logging.getLogger(__name__)
 
+DISK_SPACE_TEXT = """
+Disk space requirements
+-----------------------
+v3bw files: the maximum space required is ~{mb_bw} MB, after {d_bw} days.
+result files: the maximum space required is ~{mb_results} MB, after {d_r} days.
+tor directory: the space required is ~{mb_tor} MB.
+code and depenencies: the space required is ~{mb_code} MB
+Total disk space required is: ~{mb_total} MB
+"""
+
 
 def sbws_required_disk_space(conf):
     """Disk space required by sbws files.
     Rough calculations.
 
     :param ConfigParser conf: sbws configuration
-    :returns: int, size in MiB
+    :returns: int, size in MB
     """
+    text_dict = {}
     # Number of relays per line average size in Bytes
     size_v3bw_file = 7500 * 220
-    # the minimum number of required v3bw files
-    num_v3bw_files = 2
+    # default crontab configuration will run genenerate every hour
+    num_v3bw_files_day = 24
     # ~1000 is the length of a line when the result is successfull
     # ~4550 is the number of lines of the biggest result file
     size_result_file = 4550 * 1000
-    num_result_files = conf.getint('general', 'data_period')
-    # not counting compressed files
-    space_v3bw_files = size_v3bw_file * num_v3bw_files
-    space_result_files = size_result_file * num_result_files
+    num_result_files_day = 1
+    space_v3bw_files_day = size_v3bw_file * num_v3bw_files_day
+    space_result_files_day = size_result_file * num_result_files_day
+    size_compressed_files = 600 * 1024
+    # default crontab configuration will run cleanup once a day
+    # default cleanup configuration will compress v3bw files after 1 day
+    # and delete them after 7 days
+    v3bw_compress_after_days = conf.getint('cleanup',
+                                           'v3bw_files_compress_after_days')
+    v3bw_delete_after_days = conf.getint('cleanup',
+                                         'v3bw_files_delete_after_days')
+    v3bw_max_space_after_delete = \
+        (space_v3bw_files_day * v3bw_compress_after_days) + \
+        (size_compressed_files * num_v3bw_files_day * v3bw_delete_after_days)
+    text_dict['mb_bw'] = round(v3bw_max_space_after_delete / 1000 ** 2)
+    text_dict['d_bw'] = v3bw_delete_after_days
+    # default crontab configuration will run cleanup once a day
+    # default cleanup configuration will compress v3bw files after 1 day
+    # and delete them after 7 days
+    results_compress_after_days = conf.getint('cleanup',
+                                              'data_files_compress_after_days')
+    results_delete_after_days = conf.getint('cleanup',
+                                            'data_files_delete_after_days')
+    results_max_space_after_delete = \
+        (space_result_files_day * results_compress_after_days) + \
+        (size_compressed_files * num_v3bw_files_day *
+         results_delete_after_days)
+    text_dict['mb_results'] = round(results_max_space_after_delete / 1000 ** 2)
+    text_dict['d_r'] = results_delete_after_days
     # not counted rotated files and assuming that when it is not rotated the
     # size will be aproximately 10MiB
-    size_log_file = conf.getint('logging', 'to_file_max_bytes') or 10485760 \
-        if conf.getboolean('logging', 'to_file') else 0
+    space_log_files = 0
+    if conf.getboolean('logging', 'to_file'):
+        size_log_file = conf.getint('logging', 'to_file_max_bytes')
+        num_log_files = conf.geting('logging', 'to_file_num_backups')
+        space_log_files = size_log_file * num_log_files
+    text_dict['mb_log'] = space_log_files
     # roughly, size of a current tor dir
-    space_tor_dir = 19828000
-    # duplicate everything to warn early
-    size_total = (space_v3bw_files + size_log_file + space_result_files +
-                  space_tor_dir) * 2
-    # convert to MiB
-    size_total_mb = round(size_total / (1024 ** 2))
-    return size_total_mb
+    size_tor_dir = 19828000
+    text_dict['mb_tor'] = round(size_tor_dir / 1000 ** 2)
+    # roughly, the size of this code and dependencies
+    size_code_deps = 2097152
+    text_dict['mb_code'] = round(size_code_deps / 1000 ** 2)
+    # Multiply per 2, just in case
+    size_total = (results_max_space_after_delete +
+                  v3bw_max_space_after_delete + space_log_files + size_tor_dir +
+                  size_code_deps) * 2
+    text_dict['mb_total'] = round(size_total / 1000 ** 2)
+    space_text = DISK_SPACE_TEXT.format(**text_dict)
+    return space_text
 
 
 def df(path):