[tor-commits] [gettor/master] Rewrite bundles2github to use releases
ilv at torproject.org
ilv at torproject.org
Mon Feb 15 20:06:58 UTC 2016
commit 678bf6c64bc8a77c4351e35341159b6b543399cb
Author: Donncha O'Cearbhaill <donncha at donncha.is>
Date: Sat Feb 6 02:27:26 2016 +0100
Rewrite bundles2github to use releases
---
gettor/utils.py | 86 ++++++++++-----------
upload/bundles2github.py | 190 ++++++++++++++++++++++++-----------------------
upload/github.cfg | 8 ++
3 files changed, 142 insertions(+), 142 deletions(-)
diff --git a/gettor/utils.py b/gettor/utils.py
index c684097..c4f43c7 100644
--- a/gettor/utils.py
+++ b/gettor/utils.py
@@ -11,6 +11,7 @@
# :license: This is Free Software. See LICENSE for license information.
import os
+import re
import hashlib
"""Common utilities for GetTor modules."""
@@ -19,6 +20,10 @@ import hashlib
LOGGING_FORMAT = "[%(levelname)s] %(asctime)s; %(message)s"
DATE_FORMAT = "%Y-%m-%d" # %H:%M:%S
+windows_regex = '^torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe$'
+linux_regex = '^tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz$'
+osx_regex = '^TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg$'
+
def get_logging_format():
"""Get the logging format.
@@ -49,7 +54,7 @@ def get_sha256(string):
return str(hashlib.sha256(string).hexdigest())
-def get_bundle_info(file, osys):
+def get_bundle_info(filename, osys=None):
"""Get the os, arch and lc from a bundle string.
:param: file (string) the name of the file.
@@ -60,63 +65,35 @@ def get_bundle_info(file, osys):
:return: (list) the os, arch and lc.
"""
- if(osys == 'windows'):
- m = re.search(
- 'torbrowser-install-\d\.\d\.\d_(\w\w)(-\w\w)?\.exe',
- file)
- if m:
- lc = m.group(1)
- return 'windows', '32/64', lc
- else:
- raise ValueError("Invalid bundle format %s" % file)
- elif(osys == 'linux'):
- m = re.search(
- 'tor-browser-linux(\d\d)-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
- file)
- if m:
- arch = m.group(1)
- lc = m.group(2)
- return 'linux', arch, lc
- else:
- raise ValueError("Invalid bundle format %s" % file)
- elif(osys == 'osx'):
- m = re.search(
- 'TorBrowser-\d\.\d\.\d-osx(\d\d)_(\w\w)(-\w\w)?\.dmg',
- file)
- if m:
- os = 'osx'
- arch = m.group(1)
- lc = m.group(2)
- return 'osx', arch, lc
- else:
- raise ValueError("Invalid bundle format %s" % file)
-
-
-def valid_format(file, osys):
+ m_windows = re.search(windows_regex, filename)
+ m_linux = re.search(linux_regex, filename)
+ m_osx = re.search(osx_regex, filename)
+
+ if m_windows:
+ return 'windows', '32/64', m_windows.group(1)
+ elif m_linux:
+ return 'linux', m_linux.group(1), m_linux.group(2)
+ elif m_osx:
+ return 'osx', m_osx.group(1), m_osx.group(2)
+ else:
+ raise ValueError("Invalid bundle format %s" % file)
+
+
+def valid_format(filename, osys=None):
"""Check for valid bundle format
Check if the given file has a valid bundle format
(e.g. tor-browser-linux32-3.6.2_es-ES.tar.xz)
:param: file (string) the name of the file.
- :param: osys (string) the OS.
:return: (boolean) true if the bundle format is valid, false otherwise.
"""
- if(osys == 'windows'):
- m = re.search(
- 'torbrowser-install-\d\.\d\.\d_\w\w(-\w\w)?\.exe',
- file)
- elif(osys == 'linux'):
- m = re.search(
- 'tor-browser-linux\d\d-\d\.\d\.\d_(\w\w)(-\w\w)?\.tar\.xz',
- file)
- elif(osys == 'osx'):
- m = re.search(
- 'TorBrowser-\d\.\d\.\d-osx\d\d_(\w\w)(-\w\w)?\.dmg',
- file)
- if m:
+ m_windows = re.search(windows_regex, filename)
+ m_linux = re.search(linux_regex, filename)
+ m_osx = re.search(osx_regex, filename)
+ if any((m_windows, m_linux, m_osx)):
return True
else:
return False
@@ -139,3 +116,16 @@ def get_file_sha256(file):
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
return hasher.hexdigest()
+
+
+def find_files_to_upload(upload_dir):
+ """
+ Find the files which are named correctly and have a .asc file
+ """
+ files = []
+ for name in os.listdir(upload_dir):
+ asc_file = os.path.join(upload_dir, "{}.asc".format(name))
+ if valid_format(name) and os.path.isfile(asc_file):
+ files.extend([name, "{}.asc".format(name)])
+
+ return files
diff --git a/upload/bundles2github.py b/upload/bundles2github.py
index decb44e..8668c78 100644
--- a/upload/bundles2github.py
+++ b/upload/bundles2github.py
@@ -13,24 +13,51 @@
import os
import re
-import sh
import sys
-import time
-import shutil
-import hashlib
import argparse
+import ConfigParser
-from libsaas.services import github
+import github3
import gnupg
import gettor.core
-from gettor.utils import get_bundle_info, get_file_sha256
+from gettor.utils import (get_bundle_info, get_file_sha256,
+ find_files_to_upload)
+
+import urllib3
+
+# Actually verify Github's cert!
+urllib3.disable_warnings()
+
+
+def upload_new_release(github_repo, version, upload_dir):
+ """
+ Returns a Release object
+ """
+
+ # Create a new release of this TBB
+ release = target_repo.create_release(
+ 'v{}'.format(version),
+ target_commitish="master",
+ name='Tor Browser Bundle {}'.format(version),
+ body='',
+ draft=True,
+ )
+
+ for filename in find_files_to_upload(upload_dir):
+ # Upload each file for this release
+ file_path = os.path.join(upload_dir, filename)
+ print("Uploading file {}".format(filename))
+ release.upload_asset('application/zip',
+ filename, open(file_path, 'rb'))
+
+ return release
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Utility to upload Tor Browser to Github.'
)
-
+
# with this we only get the links of files already uploaded
# useful when somethings fail after uploading
parser.add_argument(
@@ -45,24 +72,33 @@ if __name__ == '__main__':
args = parser.parse_args()
+ config = ConfigParser.ConfigParser()
+ config.read('github-local.cfg')
+
# this script should be called after fetching the latest Tor Browser,
# and specifying the latest version
- version = args.version
+ if args.version:
+ version = args.version
+ else:
+ tbb_version_config = ConfigParser.ConfigParser()
+ tbb_version_config.read('latest_torbrowser.cfg')
+ version = tbb_version_config.get('version', 'current')
# the token allow us to run this script without GitHub user/pass
- gh_token = ''
+ github_access_token = config.get('app', 'access_token')
# path to the fingerprint that signed the packages
- tb_key = os.path.abspath('torbrowser-key.asc')
+ tb_key = os.path.abspath('tbb-key-torbrowserteam.asc')
# path to the latest version of Tor Browser
- tb_path = os.path.abspath('upload/latest')
+ tb_path = os.path.abspath('latest')
- # path to the repository where we upload Tor Browser
- repo_path = os.path.abspath('gettorbrowser')
+ # user and repository where we upload Tor Browser
+ github_user = config.get('app', 'user')
+ github_repo = config.get('app', 'repo')
- # wait time between pushing the files to GH and asking for its links
- wait_time = 10
+ gh = github3.login(token=github_access_token)
+ target_repo = gh.repository(github_user, github_repo)
# import key fingerprint
gpg = gnupg.GPG()
@@ -74,86 +110,52 @@ if __name__ == '__main__':
# e.g. 123A 456B 789C 012D 345E 678F 901G 234H 567I 890J
readable_fp = ' '.join(fp[i:i+4] for i in xrange(0, len(fp), 4))
- # we should have previously created a repository on GitHub where we
- # want to push the files using an SSH key (to avoid using user/pass)
- remote = 'origin'
- branch = 'master'
- user = 'TheTorProject'
- repo = 'gettorbrowser'
- raw_content = 'https://raw.githubusercontent.com/%s/%s/%s/' %\
- (user, repo, branch)
-
- # steps:
- # 1) copy folder with latest version of Tor Browser
- # 2) add files via sh.git
- # 3) make a commit for the new version
- # 4) push the changes
-
- if not args.links:
- shutil.copytree(
- tb_path,
- os.path.abspath('%s/%s' % (repo_path, version))
- )
-
- git = sh.git.bake(_cwd=repo_path)
- git.add('%s' % version)
- git.commit(m=version)
- git.push()
-
- # it takes a while to process the recently pushed files
- print "Wait a few seconds before asking for the links to Github..."
- time.sleep(wait_time)
-
- print "Trying to get the links"
- gh = github.GitHub(gh_token, None)
- repocontent = gh.repo(
- user,
- repo
- ).contents().get('%s' % version)
-
- core = gettor.core.Core(
- os.path.abspath('core.cfg')
- )
-
- # erase old links, if any
+ # Find any published releases with this version number
+ for release in target_repo.iter_releases():
+ if release.tag_name == 'v{}'.format(version) and not release.draft:
+ print("Found an existing published release with this version. "
+ "Not uploading again unless you delete the published "
+ "release '{}'.".format(release.tag_name))
+ break
+ else:
+ release = None
+
+ if args.links or release:
+ # Only generating link file, should use previously published release
+ if not release:
+ print("Error occured! Could not find a published release for "
+ "version {}".format(version))
+ sys.exit(1)
+
+ else:
+ # Remove any drafts to clean broken uploads
+ print('Uploading release, please wait, this might take a while!')
+ # Upload the latest browser bundles to a new release
+ release = upload_new_release(target_repo, version, tb_path)
+
+ # Upload success, publish the release
+ release.edit(draft=False)
+
+ # Create the links file for this release
+ core = gettor.core.Core(os.path.abspath('../core.cfg'))
+
+ # Erase old links if any and create a new empty one
core.create_links_file('GitHub', readable_fp)
- for file in repocontent:
- # e.g. https://raw.githubusercontent.com/gettorbrowser/dl/master/4.0.7/TorBrowser-4.0.4-osx32_en-US.dmg
- m = re.search('%s.*\/(.*)' % raw_content, file[u'download_url'])
- if m:
- filename = m.group(1)
- # get bundle info according to its OS
- if re.match('.*\.exe$', filename):
- osys, arch, lc = get_bundle_info(filename, 'windows')
- filename_asc = filename.replace('exe', 'exe.asc')
-
- elif re.match('.*\.dmg$', filename):
- osys, arch, lc = get_bundle_info(filename, 'osx')
- filename_asc = filename.replace('dmg', 'dmg.asc')
-
- elif re.match('.*\.tar.xz$', filename):
- osys, arch, lc = get_bundle_info(filename, 'linux')
- filename_asc = filename.replace('tar.xz', 'tar.xz.asc')
-
- else:
- # don't care about other files (asc or txt)
- continue
-
- sha256 = get_file_sha256(
- os.path.abspath(
- '%s/%s/%s' % (repo, version, filename)
- )
- )
-
- # since the url is easy to construct and it doesn't involve any
- # kind of unique hash or identifier, we get the link for the
- # asc signature just by adding '.asc'
- link_asc = file[u'download_url'].replace(filename, filename_asc)
-
- link = "%s$%s$%s$" % (file[u'download_url'], link_asc, sha256)
-
- print "Adding %s" % file[u'download_url']
- core.add_link('GitHub', osys, lc, link)
+ print("Creating releases")
+ for asset in release.assets:
+ filename = asset.download_url.split('/')[-1]
+ osys, arch, lc = get_bundle_info(asset.download_url)
+ sha256 = get_file_sha256(
+ os.path.abspath(os.path.join(tb_path, filename))
+ )
+
+ link = "{}${}${}$".format(
+ asset.download_url,
+ asset.download_url+".asc",
+ sha256,
+ )
+ print("Adding {}".format(asset.download_url))
+ core.add_link('GitHub', osys, lc, link)
print "Github links updated!"
diff --git a/upload/github.cfg b/upload/github.cfg
new file mode 100644
index 0000000..7bdbd90
--- /dev/null
+++ b/upload/github.cfg
@@ -0,0 +1,8 @@
+[general]
+upload_dir: latest
+tbb_key: torbrowser-key.asc
+
+[app]
+access_token: suchtoken
+user: username
+repo: gettor-front
More information about the tor-commits
mailing list