[tor-commits] [ooni-probe/master] Remove dead code
art at torproject.org
art at torproject.org
Wed Nov 21 15:36:19 UTC 2012
commit a194337696e30a9efd0328d9a4943cfcd26ca571
Author: Arturo Filastò <art at fuffa.org>
Date: Tue Nov 20 21:04:32 2012 +0100
Remove dead code
---
to-be-ported/old-api/.ropeproject/config.py | 85 ---------
to-be-ported/old-api/.ropeproject/globalnames | Bin 108 -> 0 bytes
to-be-ported/old-api/.ropeproject/history | 1 -
to-be-ported/old-api/.ropeproject/objectdb | Bin 741 -> 0 bytes
to-be-ported/old-api/TESTS_ARE_MOVING.txt | 8 -
to-be-ported/old-api/domclass.py | 216 ----------------------
to-be-ported/old-api/dropin.cache | 243 -------------------------
to-be-ported/old-api/tcpscan.py | 84 ---------
8 files changed, 0 insertions(+), 637 deletions(-)
diff --git a/to-be-ported/old-api/.ropeproject/config.py b/to-be-ported/old-api/.ropeproject/config.py
deleted file mode 100644
index ffebcd4..0000000
--- a/to-be-ported/old-api/.ropeproject/config.py
+++ /dev/null
@@ -1,85 +0,0 @@
-# The default ``config.py``
-
-
-def set_prefs(prefs):
- """This function is called before opening the project"""
-
- # Specify which files and folders to ignore in the project.
- # Changes to ignored resources are not added to the history and
- # VCSs. Also they are not returned in `Project.get_files()`.
- # Note that ``?`` and ``*`` match all characters but slashes.
- # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc'
- # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc'
- # '.svn': matches 'pkg/.svn' and all of its children
- # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o'
- # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o'
- prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject',
- '.hg', '.svn', '_svn', '.git']
-
- # Specifies which files should be considered python files. It is
- # useful when you have scripts inside your project. Only files
- # ending with ``.py`` are considered to be python files by
- # default.
- #prefs['python_files'] = ['*.py']
-
- # Custom source folders: By default rope searches the project
- # for finding source folders (folders that should be searched
- # for finding modules). You can add paths to that list. Note
- # that rope guesses project source folders correctly most of the
- # time; use this if you have any problems.
- # The folders should be relative to project root and use '/' for
- # separating folders regardless of the platform rope is running on.
- # 'src/my_source_folder' for instance.
- #prefs.add('source_folders', 'src')
-
- # You can extend python path for looking up modules
- #prefs.add('python_path', '~/python/')
-
- # Should rope save object information or not.
- prefs['save_objectdb'] = True
- prefs['compress_objectdb'] = False
-
- # If `True`, rope analyzes each module when it is being saved.
- prefs['automatic_soa'] = True
- # The depth of calls to follow in static object analysis
- prefs['soa_followed_calls'] = 0
-
- # If `False` when running modules or unit tests "dynamic object
- # analysis" is turned off. This makes them much faster.
- prefs['perform_doa'] = True
-
- # Rope can check the validity of its object DB when running.
- prefs['validate_objectdb'] = True
-
- # How many undos to hold?
- prefs['max_history_items'] = 32
-
- # Shows whether to save history across sessions.
- prefs['save_history'] = True
- prefs['compress_history'] = False
-
- # Set the number spaces used for indenting. According to
- # :PEP:`8`, it is best to use 4 spaces. Since most of rope's
- # unit-tests use 4 spaces it is more reliable, too.
- prefs['indent_size'] = 4
-
- # Builtin and c-extension modules that are allowed to be imported
- # and inspected by rope.
- prefs['extension_modules'] = []
-
- # Add all standard c-extensions to extension_modules list.
- prefs['import_dynload_stdmods'] = True
-
- # If `True` modules with syntax errors are considered to be empty.
- # The default value is `False`; When `False` syntax errors raise
- # `rope.base.exceptions.ModuleSyntaxError` exception.
- prefs['ignore_syntax_errors'] = False
-
- # If `True`, rope ignores unresolvable imports. Otherwise, they
- # appear in the importing namespace.
- prefs['ignore_bad_imports'] = False
-
-
-def project_opened(project):
- """This function is called after opening the project"""
- # Do whatever you like here!
diff --git a/to-be-ported/old-api/.ropeproject/globalnames b/to-be-ported/old-api/.ropeproject/globalnames
deleted file mode 100644
index 2877ef5..0000000
Binary files a/to-be-ported/old-api/.ropeproject/globalnames and /dev/null differ
diff --git a/to-be-ported/old-api/.ropeproject/history b/to-be-ported/old-api/.ropeproject/history
deleted file mode 100644
index fcd9c96..0000000
--- a/to-be-ported/old-api/.ropeproject/history
+++ /dev/null
@@ -1 +0,0 @@
-€]q(]q]qe.
\ No newline at end of file
diff --git a/to-be-ported/old-api/.ropeproject/objectdb b/to-be-ported/old-api/.ropeproject/objectdb
deleted file mode 100644
index f276839..0000000
Binary files a/to-be-ported/old-api/.ropeproject/objectdb and /dev/null differ
diff --git a/to-be-ported/old-api/TESTS_ARE_MOVING.txt b/to-be-ported/old-api/TESTS_ARE_MOVING.txt
deleted file mode 100644
index f4c0084..0000000
--- a/to-be-ported/old-api/TESTS_ARE_MOVING.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-7/10/2012
-
-All new tests will be moved to the directory /nettests/.
-
-Tests that are in this directory are either here for historical reasons or have
-not yet been properly tested and fully supporting the new API.
-
-A.
diff --git a/to-be-ported/old-api/domclass.py b/to-be-ported/old-api/domclass.py
deleted file mode 100644
index 3080c40..0000000
--- a/to-be-ported/old-api/domclass.py
+++ /dev/null
@@ -1,216 +0,0 @@
-#!/usr/bin/env python
-#-*- encoding: utf-8 -*-
-#
-# domclass
-# ********
-#
-# :copyright: (c) 2012 by Arturo Filastò
-# :license: see LICENSE for more details.
-#
-# how this works
-# --------------
-#
-# This classifier uses the DOM structure of a website to determine how similar
-# the two sites are.
-# The procedure we use is the following:
-# * First we parse all the DOM tree of the web page and we build a list of
-# TAG parent child relationships (ex. <html><a><b></b></a><c></c></html> =>
-# (html, a), (a, b), (html, c)).
-#
-# * We then use this information to build a matrix (M) where m[i][j] = P(of
-# transitioning from tag[i] to tag[j]). If tag[i] does not exists P() = 0.
-# Note: M is a square matrix that is number_of_tags wide.
-#
-# * We then calculate the eigenvectors (v_i) and eigenvalues (e) of M.
-#
-# * The corelation between page A and B is given via this formula:
-# correlation = dot_product(e_A, e_B), where e_A and e_B are
-# resepectively the eigenvalues for the probability matrix A and the
-# probability matrix B.
-#
-
-try:
- import numpy
-except:
- print "Error numpy not installed!"
-
-import yaml
-from zope.interface import implements
-from twisted.python import usage
-from twisted.plugin import IPlugin
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset
-from ooni.utils import log
-from ooni.protocols.http import HTTPTest
-
-class domclassArgs(usage.Options):
- optParameters = [['output', 'o', None, 'Output to write'],
- ['file', 'f', None, 'Corpus file'],
- ['fileb', 'b', None, 'Corpus file'],
- ['urls', 'u', None, 'URL List'],
- ['resume', 'r', 0, 'Resume at this index']]
-
-# All HTML4 tags
-# XXX add link to W3C page where these came from
-alltags = ['A', 'ABBR', 'ACRONYM', 'ADDRESS', 'APPLET', 'AREA', 'B', 'BASE',
- 'BASEFONT', 'BD', 'BIG', 'BLOCKQUOTE', 'BODY', 'BR', 'BUTTON', 'CAPTION',
- 'CENTER', 'CITE', 'CODE', 'COL', 'COLGROUP', 'DD', 'DEL', 'DFN', 'DIR', 'DIV',
- 'DL', 'DT', 'E M', 'FIELDSET', 'FONT', 'FORM', 'FRAME', 'FRAMESET', 'H1', 'H2',
- 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'I', 'IFRAME ', 'IMG',
- 'INPUT', 'INS', 'ISINDEX', 'KBD', 'LABEL', 'LEGEND', 'LI', 'LINK', 'MAP',
- 'MENU', 'META', 'NOFRAMES', 'NOSCRIPT', 'OBJECT', 'OL', 'OPTGROUP', 'OPTION',
- 'P', 'PARAM', 'PRE', 'Q', 'S', 'SAMP', 'SCRIPT', 'SELECT', 'SMALL', 'SPAN',
- 'STRIKE', 'STRONG', 'STYLE', 'SUB', 'SUP', 'TABLE', 'TBODY', 'TD',
- 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'U', 'UL', 'VAR']
-
-# Reduced subset of only the most common tags
-commontags = ['A', 'B', 'BLOCKQUOTE', 'BODY', 'BR', 'BUTTON', 'CAPTION',
- 'CENTER', 'CITE', 'CODE', 'COL', 'DD', 'DIV',
- 'DL', 'DT', 'EM', 'FIELDSET', 'FONT', 'FORM', 'FRAME', 'FRAMESET', 'H1', 'H2',
- 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'IFRAME ', 'IMG',
- 'INPUT', 'INS', 'LABEL', 'LEGEND', 'LI', 'LINK', 'MAP',
- 'MENU', 'META', 'NOFRAMES', 'NOSCRIPT', 'OBJECT', 'OL', 'OPTION',
- 'P', 'PRE', 'SCRIPT', 'SELECT', 'SMALL', 'SPAN',
- 'STRIKE', 'STRONG', 'STYLE', 'SUB', 'SUP', 'TABLE', 'TBODY', 'TD',
- 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'U', 'UL']
-
-# The tags we are intested in using for our analysis
-thetags = ['A', 'DIV', 'FRAME', 'H1', 'H2',
- 'H3', 'H4', 'IFRAME ', 'INPUT',
- 'LABEL','LI', 'P', 'SCRIPT', 'SPAN',
- 'STYLE', 'TR']
-
-def compute_probability_matrix(dataset):
- """
- Compute the probability matrix based on the input dataset.
-
- :dataset: an array of pairs representing the parent child relationships.
- """
- import itertools
- ret = {}
- matrix = numpy.zeros((len(thetags) + 1, len(thetags) + 1))
-
- for data in dataset:
- x = data[0].upper()
- y = data[1].upper()
- try:
- x = thetags.index(x)
- except:
- x = len(thetags)
-
- try:
- y = thetags.index(y)
- except:
- y = len(thetags)
-
- matrix[x,y] += 1
-
- for x in xrange(len(thetags) + 1):
- possibilities = 0
- for y in matrix[x]:
- possibilities += y
-
- for i in xrange(len(matrix[x])):
- if possibilities != 0:
- matrix[x][i] = matrix[x][i]/possibilities
-
- return matrix
-
-def compute_eigenvalues(matrix):
- """
- Returns the eigenvalues of the supplied square matrix.
-
- :matrix: must be a square matrix and diagonalizable.
- """
- return numpy.linalg.eigvals(matrix)
-
-def readDOM(content=None, filename=None):
- """
- Parses the DOM of the HTML page and returns an array of parent, child
- pairs.
-
- :content: the content of the HTML page to be read.
-
- :filename: the filename to be read from for getting the content of the
- page.
- """
- from bs4 import BeautifulSoup
-
- if filename:
- f = open(filename)
- content = ''.join(f.readlines())
- f.close()
-
- dom = BeautifulSoup(content)
- couples = []
- for x in dom.findAll():
- couples.append((str(x.parent.name), str(x.name)))
-
- return couples
-
-class domclassTest(HTTPTest):
- implements(IPlugin, ITest)
-
- shortName = "domclass"
- description = "domclass"
- requirements = None
- options = domclassArgs
- blocking = False
-
- follow_redirects = True
- #tool = True
-
- def runTool(self):
- site_a = readDOM(filename=self.local_options['file'])
- site_b = readDOM(filename=self.local_options['fileb'])
- a = {}
- a['matrix'] = compute_probability_matrix(site_a)
- a['eigen'] = compute_eigenvalues(a['matrix'])
-
- self.result['eigenvalues'] = a['eigen']
- b = {}
- b['matrix'] = compute_probability_matrix(site_b)
- b['eigen'] = compute_eigenvalues(b['matrix'])
-
- #print "A: %s" % a
- #print "B: %s" % b
- correlation = numpy.vdot(a['eigen'],b['eigen'])
- correlation /= numpy.linalg.norm(a['eigen'])*numpy.linalg.norm(b['eigen'])
- correlation = (correlation + 1)/2
- print "Corelation: %s" % correlation
- self.end()
- return a
-
- def processResponseBody(self, data):
- site_a = readDOM(data)
- #site_b = readDOM(self.local_options['fileb'])
- a = {}
- a['matrix'] = compute_probability_matrix(site_a)
- a['eigen'] = compute_eigenvalues(a['matrix'])
-
-
- if len(data) == 0:
- self.result['eigenvalues'] = None
- self.result['matrix'] = None
- else:
- self.result['eigenvalues'] = a['eigen']
- #self.result['matrix'] = a['matrix']
- #self.result['content'] = data[:200]
- #b = compute_matrix(site_b)
- print "A: %s" % a
- return a['eigen']
-
- def load_assets(self):
- if self.local_options:
- if self.local_options['file']:
- self.tool = True
- return {}
- elif self.local_options['urls']:
- return {'url': Asset(self.local_options['urls'])}
- else:
- self.end()
- return {}
- else:
- return {}
-
-#domclass = domclassTest(None, None, None)
diff --git a/to-be-ported/old-api/dropin.cache b/to-be-ported/old-api/dropin.cache
deleted file mode 100755
index 65c2187..0000000
--- a/to-be-ported/old-api/dropin.cache
+++ /dev/null
@@ -1,243 +0,0 @@
-(dp1
-S'tcpconnect'
-p2
-ccopy_reg
-_reconstructor
-p3
-(ctwisted.plugin
-CachedDropin
-p4
-c__builtin__
-object
-p5
-NtRp6
-(dp7
-S'moduleName'
-p8
-S'ooni.plugins.tcpconnect'
-p9
-sS'description'
-p10
-S'\nThis is a self genrated test created by scaffolding.py.\nyou will need to fill it up with all your necessities.\nSafe hacking :).\n'
-p11
-sS'plugins'
-p12
-(lp13
-g3
-(ctwisted.plugin
-CachedPlugin
-p14
-g5
-NtRp15
-(dp16
-S'provided'
-p17
-(lp18
-ctwisted.plugin
-IPlugin
-p19
-acooni.plugoo.interface
-ITest
-p20
-asS'dropin'
-p21
-g6
-sS'name'
-p22
-S'tcpconnect'
-p23
-sg10
-NsbasbsS'domclass'
-p24
-g3
-(g4
-g5
-NtRp25
-(dp26
-g8
-S'ooni.plugins.domclass'
-p27
-sg10
-Nsg12
-(lp28
-g3
-(g14
-g5
-NtRp29
-(dp30
-g17
-(lp31
-g19
-ag20
-asg21
-g25
-sg22
-S'domclass'
-p32
-sg10
-NsbasbsS'bridget'
-p33
-g3
-(g4
-g5
-NtRp34
-(dp35
-g8
-S'ooni.plugins.bridget'
-p36
-sg10
-Nsg12
-(lp37
-g3
-(g14
-g5
-NtRp38
-(dp39
-g17
-(lp40
-g19
-ag20
-asg21
-g34
-sg22
-S'bridget'
-p41
-sg10
-S"\n XXX fill me in\n\n :ivar config:\n An :class:`ooni.lib.txtorcon.TorConfig` instance.\n :ivar relays:\n A list of all provided relays to test.\n :ivar bridges:\n A list of all provided bridges to test.\n :ivar socks_port:\n Integer for Tor's SocksPort.\n :ivar control_port:\n Integer for Tor's ControlPort.\n :ivar transport:\n String defining the Tor's ClientTransportPlugin, for testing \n a bridge's pluggable transport functionality.\n :ivar tor_binary:\n Path to the Tor binary to use, e.g. '/usr/sbin/tor'\n "
-p42
-sbasbsS'daphn3'
-p43
-g3
-(g4
-g5
-NtRp44
-(dp45
-g8
-S'plugins.daphn3'
-p46
-sg10
-S'\nThis is a self genrated test created by scaffolding.py.\nyou will need to fill it up with all your necessities.\nSafe hacking :).\n'
-p47
-sg12
-(lp48
-g3
-(g14
-g5
-NtRp49
-(dp50
-g17
-(lp51
-g19
-ag20
-asg21
-g44
-sg22
-S'daphn3test'
-p52
-sg10
-NsbasbsS'httpt'
-p53
-g3
-(g4
-g5
-NtRp54
-(dp55
-g8
-S'ooni.plugins.httpt'
-p56
-sg10
-S'\nThis is a self genrated test created by scaffolding.py.\nyou will need to fill it up with all your necessities.\nSafe hacking :).\n'
-p57
-sg12
-(lp58
-sbsS'chinatrigger'
-p59
-g3
-(g4
-g5
-NtRp60
-(dp61
-g8
-S'plugins.chinatrigger'
-p62
-sg10
-Nsg12
-(lp63
-g3
-(g14
-g5
-NtRp64
-(dp65
-g17
-(lp66
-g19
-ag20
-asg21
-g60
-sg22
-S'chinatrigger'
-p67
-sg10
-S'\n This test is a OONI based implementation of the C tool written\n by Philipp Winter to engage chinese probes in active scanning.\n\n Example of running it:\n ./ooni/ooniprobe.py chinatrigger -d 127.0.0.1 -p 8080 -f bla.pcap\n '
-p68
-sbasbsS'dnstamper'
-p69
-g3
-(g4
-g5
-NtRp70
-(dp71
-g8
-S'ooni.plugins.dnstamper'
-p72
-sg10
-S'\n dnstamper\n *********\n\n This test resolves DNS for a list of domain names, one per line, in the\n file specified in the ooni-config under the setting "dns_experiment". If\n the file is top-1m.txt, the test will be run using Amazon\'s list of top\n one million domains. The experimental dns servers to query should\n be specified one per line in assets/dns_servers.txt.\n\n The test reports censorship if the cardinality of the intersection of\n the query result set from the control server and the query result set\n from the experimental server is zero, which is to say, if the two sets\n have no matching results whatsoever.\n\n NOTE: This test frequently results in false positives due to GeoIP-based\n load balancing on major global sites such as google, facebook, and\n youtube, etc.\n\n :author: Isis Lovecruft, Arturo Filast\xc3\xb2\n :license: see LICENSE for more details\n\n TODO:\n * Finish porting to twisted\n
* Finish the client.Resolver() subclass and test it\n * Use the DNS tests from captiveportal\n * Use plugoo/reports.py for final data\n'
-p73
-sg12
-(lp74
-g3
-(g14
-g5
-NtRp75
-(dp76
-g17
-(lp77
-g19
-ag20
-asg21
-g70
-sg22
-S'dnstamper'
-p78
-sg10
-S'\n XXX fill me in\n '
-p79
-sbasbsS'blocking'
-p80
-g3
-(g4
-g5
-NtRp81
-(dp82
-g8
-S'plugins.blocking'
-p83
-sg10
-Nsg12
-(lp84
-g3
-(g14
-g5
-NtRp85
-(dp86
-g17
-(lp87
-g19
-ag20
-asg21
-g81
-sg22
-S'blocking'
-p88
-sg10
-Nsbasbs.
\ No newline at end of file
diff --git a/to-be-ported/old-api/tcpscan.py b/to-be-ported/old-api/tcpscan.py
deleted file mode 100644
index b371c88..0000000
--- a/to-be-ported/old-api/tcpscan.py
+++ /dev/null
@@ -1,84 +0,0 @@
-"""
- TCP Port Scanner
- ****************
-
- Does a TCP connect scan on the IP:port pairs.
-
-"""
-import os
-from gevent import socket
-from datetime import datetime
-import socks
-
-from plugoo.assets import Asset
-from plugoo.tests import Test
-
-__plugoo__ = "TCP Port Scanner"
-__desc__ = "This a test template to be used to build your own tests"
-
-class TCPScanAsset(Asset):
- """
- This is the asset that should be used by the Test. It will
- contain all the code responsible for parsing the asset file
- and should be passed on instantiation to the test.
- """
- def __init__(self, file=None):
- self = Asset.__init__(self, file)
-
-
-class TCPScan(Test):
- """
- The main Test class
- """
-
- def experiment(self, *a, **kw):
- """
- Fill this up with the tasks that should be performed
- on the "dirty" network and should be compared with the
- control.
- """
- addr = kw['data']
- s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- res = False
- try:
- self.logger.debug('Doing a connection to %s' % addr)
- s.connect((addr.split(':')[0], int(addr.split(':')[1])))
- res = True
- except socket.error, msg:
- self.logger.debug('Connection failed to %s: %s' % (addr, msg))
-
- finally:
- s.close()
-
- return {'Time': datetime.now(),
- 'Address': addr,
- 'Status': res}
-
- def control(self):
- """
- Fill this up with the control related code.
- """
- return True
-
-def run(ooni, asset=None):
- """
- This is the function that will be called by OONI
- and it is responsible for instantiating and passing
- the arguments to the Test class.
- """
- config = ooni.config
-
- # This the assets array to be passed to the run function of
- # the test
- if asset:
- assets = [TCPScanAsset(asset)]
- else:
- assets = [TCPScanAsset(os.path.join(config.main.assetdir, \
- "tcpscan.txt"))]
-
- # Instantiate the Test
- thetest = TCPScan(ooni)
- ooni.logger.info("starting TCP Scan...")
- # Run the test with argument assets
- thetest.run(assets)
- ooni.logger.info("finished.")
More information about the tor-commits
mailing list