[tor-commits] [ooni-probe/master] Remove geoip downloading from setup.py script

[art at torproject.org]

commit da919320ca780656e0369983d437d8e08470cf08
Author: Arturo Filastò <art at fuffa.org>
Date:   Thu Nov 13 11:42:06 2014 +0100

    Remove geoip downloading from setup.py script
---
 requirements.txt |    1 +
 setup.py         |  115 +++++++++++++++++++++++++-----------------------------
 2 files changed, 54 insertions(+), 62 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 50259ff..e28a08d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,3 +9,4 @@ parsley>=1.1
 scapy-real>=2.2.0-dev
 pypcap>=1.1
 service-identity
+pydumbnet
diff --git a/setup.py b/setup.py
index aa17f85..444bcd8 100644
--- a/setup.py
+++ b/setup.py
@@ -1,76 +1,58 @@
 #!/usr/bin/env python
-#-*- coding: utf-8 -*-
+# -*- coding: utf-8 -*-
 
 from ooni import __version__, __author__
-import urllib2
 import os
-import gzip
-from os.path import join as pj
 import sys
-from setuptools import setup
-
-def download_geoip_files(dst):
-    urls = [
-        'http://www.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz',
-        'http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz',
-        'http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz'
-    ]
-    if not os.path.exists(dst):
-        os.makedirs(dst)
-    for url in urls:
-        target_gz_file = pj(dst, os.path.basename(url))
-        target_file = target_gz_file.replace('.gz', '')
-
-        if os.path.isfile(target_file):
-            print "%s already exists. Skipping." % target_file
-            continue
-
-        print "Downloading %s" % url
-        response = urllib2.urlopen(url)
-
-        CHUNK = 4 * 1024
-        with open(target_gz_file, 'w+') as f:
-            while True:
-                chunk = response.read(CHUNK)
-                if not chunk: break
-                f.write(chunk)
-
-        with open(target_file, 'w+') as f:
-            gf = gzip.open(target_gz_file, 'rb')
-            while True:
-                chunk = gf.read(CHUNK)
-                if not chunk: break
-                f.write(chunk)
-            gf.close()
-
-        os.unlink(target_gz_file)
+import tempfile
 
-usr_share_path = '/usr/share'
-# If this is true then it means we are in a virtualenv
-# therefore we should not place our data files inside /usr/share/ooni, but
-# place them inside the virtual env system prefix.
-if hasattr(sys, 'real_prefix'):
-    usr_share_path = os.path.abspath(pj(sys.prefix, 'share'))
+from os.path import join as pj
+from setuptools import setup
+from setuptools.command.install import install as _st_install
+
+
+class install(_st_install):
+    def gen_config(self, share_path):
+        config_file = pj(tempfile.mkdtemp(), "ooniprobe.conf.sample")
+        o = open(config_file, "w+")
+        with open("data/ooniprobe.conf.sample") as f:
+            for line in f:
+                if "/usr/share" in line:
+                    line = line.replace("/usr/share", share_path)
+                o.write(line)
+        o.close()
+        return config_file
+
+    def set_data_files(self, share_path):
+        for root, dirs, file_names in os.walk('data/'):
+            files = []
+            for file_name in file_names:
+                if file_name.endswith('.pyc'):
+                    continue
+                elif file_name.endswith('.dat') and \
+                        file_name.startswith('Geo'):
+                    continue
+                elif file_name == "ooniprobe.conf.sample":
+                    files.append(self.gen_config(share_path))
+                    continue
+                files.append(pj(root, file_name))
+            self.distribution.data_files.append(
+                [
+                    pj(share_path, 'ooni', root.replace('data/', '')),
+                    files
+                ]
+            )
+
+    def run(self):
+        share_path = os.path.abspath(pj(self.prefix, 'share'))
+        self.set_data_files(share_path)
+        self.do_egg_install()
 
-download_geoip_files(pj(usr_share_path, 'GeoIP'))
 
 install_requires = []
 dependency_links = []
 data_files = []
-
-for root, dirs, file_names in os.walk('data/'):
-    files = []
-    for file_name in file_names:
-        if file_name.endswith('.pyc'):
-            continue
-        elif file_name.endswith('.dat') and \
-                file_name.startswith('Geo'):
-            continue
-        files.append(pj(root, file_name))
-    data_files.append([pj(usr_share_path, 'ooni', root.replace('data/', '')), files])
-
-
-packages=[
+packages = [
     'ooni',
     'ooni.api',
     'ooni.deckgen',
@@ -120,6 +102,7 @@ setup(
              "bin/oonireport", "bin/ooniresources"],
     dependency_links=dependency_links,
     install_requires=install_requires,
+    cmdclass={"install": install},
     classifiers=(
         "Development Status :: 5 - Production/Stable",
         "Environment :: Console",
@@ -155,3 +138,11 @@ setup(
         "Topic :: System :: Networking :: Monitoring",
     )
 )
+
+from subprocess import Popen, PIPE
+process = Popen(['ooniresources', '--update-inputs', '--update-geoip'],
+                stdout=PIPE, stderr=PIPE)
+while process.poll() is None:
+    out = process.stdout.read()
+    sys.stdout.write(out)
+    sys.stdout.flush()