[tor-commits] [ooni-probe/master] Make improvements to the setup.py install script.

[art at torproject.org]

commit b58085b41a655d0b030e5df9d6abac2efa3357e3
Author: Arturo Filastò <art at fuffa.org>
Date:   Fri Mar 14 17:21:46 2014 +0100

    Make improvements to the setup.py install script.
    
    * Download the GeoIP data files directly into the /usr/share directory
    * Move the logic for overriding the config file keys to settings.py
---
 ooni/settings.py |   14 +++++++++++-
 setup.py         |   62 ++++++++++++++++++------------------------------------
 2 files changed, 34 insertions(+), 42 deletions(-)

diff --git a/ooni/settings.py b/ooni/settings.py
index c1071b9..1c4f42d 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -69,7 +69,19 @@ class OConfig(object):
         target_config_file = os.path.join(self.ooni_home,
                                           'ooniprobe.conf')
         print "Creating it for you in '%s'." % target_config_file
-        copyfile(sample_config_file, target_config_file)
+        usr_share_path = '/usr/share'
+        if hasattr(sys, 'real_prefix'):
+            usr_share_path = os.path.abspath(os.path.join(sys.prefix, 'share'))
+        
+        with open(sample_config_file) as f:
+            with open(target_config_file, 'w+') as w:
+                for line in f:
+                    if line.startswith('    data_dir: '):
+                        w.write('    data_dir: %s\n' % os.path.join(usr_share_path, 'ooni'))
+                    elif line.startswith('    geoip_data_dir: '):
+                        w.write('    geoip_data_dir: %s\n' % os.path.join(usr_share_path, 'GeoIP'))
+                    else:
+                        w.write(line)
 
     def read_config_file(self):
         try:
diff --git a/setup.py b/setup.py
index 1410dd7..9fff96c 100644
--- a/setup.py
+++ b/setup.py
@@ -9,14 +9,16 @@ from os.path import join as pj
 import sys
 from setuptools import setup
 
-def download_geoip_files():
+def download_geoip_files(dst):
     urls = [
         'http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz',
         'http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz',
         'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
     ]
+    if not os.path.exists(dst):
+        os.makedirs(dst)
     for url in urls:
-        target_gz_file = pj('data', os.path.basename(url))
+        target_gz_file = pj(dst, os.path.basename(url))
         target_file = target_gz_file.replace('.gz', '')
 
         if os.path.isfile(target_file):
@@ -26,55 +28,35 @@ def download_geoip_files():
         print "Downloading %s" % url
         response = urllib2.urlopen(url)
 
+        CHUNK = 4 * 1024
         with open(target_gz_file, 'w+') as f:
-            f.write(response.read())
+            while True:
+                chunk = response.read(CHUNK)
+                if not chunk: break
+                f.write(chunk)
         
         with open(target_file, 'w+') as f:
             gf = gzip.open(target_gz_file, 'rb')
-            f.write(gf.read())
+            while True:
+                chunk = gf.read(CHUNK)
+                if not chunk: break
+                f.write(chunk)
             gf.close()
 
         os.unlink(target_gz_file)
 
-download_geoip_files()
-
-usr_share_path = '/usr/share/ooni'
+usr_share_path = '/usr/share'
 # If this is true then it means we are in a virtualenv
 # therefore we should not place our data files inside /usr/share/ooni, but
 # place them inside the virtual env system prefix.
 if hasattr(sys, 'real_prefix'):
-    usr_share_path = os.path.abspath(pj(sys.prefix, 'share', 'ooni'))
-    if not os.path.isdir(usr_share_path):
-        os.makedirs(usr_share_path)
-    with open(pj('data', 'ooniprobe.conf.sample.new'), 'w+') as w:
-        with open(pj('data', 'ooniprobe.conf.sample')) as f:
-            for line in f:
-                if line.startswith('    data_dir: '):
-                    w.write('    data_dir: %s\n' % usr_share_path)
-                elif line.startswith('    geoip_data_dir: '):
-                    w.write('    geoip_data_dir: %s\n' % usr_share_path)
-                else:
-                    w.write(line)
-    os.rename(pj('data', 'ooniprobe.conf.sample.new'),
-              pj('data', 'ooniprobe.conf.sample'))
+    usr_share_path = os.path.abspath(pj(sys.prefix, 'share'))
+
+download_geoip_files(pj(usr_share_path, 'GeoIP'))
 
-    data_files = [(
-        usr_share_path + '/', 
-        [
-            'data/GeoIP.dat',
-            'data/GeoIPASNum.dat',
-            'data/GeoLiteCity.dat'
-        ]
-    )]
-else:
-    data_files = [(
-        '/usr/share/ooni/geoip/', 
-        [
-            'data/GeoIP.dat',
-            'data/GeoIPASNum.dat',
-            'data/GeoLiteCity.dat'
-        ]
-    )]
+install_requires = []
+dependency_links = []
+data_files = []
 
 for root, dirs, file_names in os.walk('data/'):
     files = []
@@ -85,10 +67,8 @@ for root, dirs, file_names in os.walk('data/'):
                 file_name.startswith('Geo'):
             continue
         files.append(pj(root, file_name))
-    data_files.append([pj(usr_share_path, root.replace('data/', '')), files])
+    data_files.append([pj(usr_share_path, 'ooni', root.replace('data/', '')), files])
 
-install_requires = []
-dependency_links = []
 with open('requirements.txt') as f:
     for line in f:
         if line.startswith("#"):