[tbb-commits] [tor-browser-bundle/master] Add tools directory with scripts for nightly builds.

[mikeperry at torproject.org]

commit 784bac5f84c7b4b02e11e74070d57c944d1dc291
Author: Linus Nordberg <linus at nordberg.se>
Date:   Sat Mar 15 16:42:50 2014 +0100

    Add tools directory with scripts for nightly builds.
---
 tools/continuous-builds/README                    |   14 ++++++
 tools/continuous-builds/build-tbb.sh              |   48 +++++++++++++++++++++
 tools/continuous-builds/checkout-tbb.sh           |    8 ++++
 tools/continuous-builds/park-nightly.sh           |   42 ++++++++++++++++++
 tools/continuous-builds/setup-gitian-build-env.sh |    3 ++
 5 files changed, 115 insertions(+)

diff --git a/tools/continuous-builds/README b/tools/continuous-builds/README
new file mode 100644
index 0000000..c60f19f
--- /dev/null
+++ b/tools/continuous-builds/README
@@ -0,0 +1,14 @@
+Nightly builds can be produced and uploaded to a publishing host by
+running build-tbb.sh. The checkout-tbb.sh script can be used to check
+out the latest master before starting a build.
+
+The publishing host should handle uploaded tar file using the
+park-nightly.sh script through the command option in sshd's
+AuthorizedKeysFile.
+
+  command="[ $(hostname) = perdulce ] && ~/park-nightly.sh",no-pty ssh-rsa ...
+
+In order to not grow the disk usage too much on the publishing host,
+the following script can be run by cron:
+https://github.com/boklm/prune-old-builds.
+
diff --git a/tools/continuous-builds/build-tbb.sh b/tools/continuous-builds/build-tbb.sh
new file mode 100755
index 0000000..c44e0fd
--- /dev/null
+++ b/tools/continuous-builds/build-tbb.sh
@@ -0,0 +1,48 @@
+#! /bin/sh
+# usage:
+# build-tbb.sh [TARGET [PUBLISH-HOST [PUBLISH-KEY [BUILDDIR [DESTDIR [N]]]]]]
+#
+# Build TARGET in BUILDDIR, which will end up in DESTDIR
+# Try doing it N times.
+# Upload result to PUBLISH-HOST using SSH key PUBLISH-KEY.
+
+# TODO:
+# - if there's no new commits, don't build but make sure there's a symlink on perdulce
+
+TARGET=$1; [ -z "$TARGET" ] && TARGET=nightly
+PUBLISH_HOST=$2; [ -z "$PUBLISH_HOST" ] && PUBLISH_HOST=perdulce.torproject.org
+PUBLISH_KEY=$3; [ -z "$PUBLISH_KEY" ] && PUBLISH_KEY=~/.ssh/perdulce-upload
+BUILDDIR=$4; [ -z "$BUILDDIR" ] && BUILDDIR=~/usr/src/tor-browser-bundle/gitian
+DESTDIR=$5; [ -z "$DESTDIR" ] && DESTDIR=$BUILDDIR/tbb-$TARGET
+N=$6; [ -z "$N" ] && N=16
+
+logfile=$(date -u +%s).log
+
+. ~/setup-gitian-build-env.sh
+cd $BUILDDIR || exit 1
+status=init
+n=0
+while [ $status != done ]; do
+  n=$(expr $n + 1)
+  printf "%s: Starting build number %d. target=$TARGET.\n" $0 $n | tee -a $logfile
+  date | tee -a $logfile
+  killall qemu-system-i386 qemu-system-x86_64
+  make $TARGET > build-$(date -u +%s).log && status=done
+  printf "%s: Tried building $TARGET %d times. Status: %s.\n" $0 $n $status | tee -a $logfile
+  TARGET=build-nightly
+  [ $n -ge $N ] && break
+done
+
+if [ $status = done ]; then
+  NEWDESTDIR=$DESTDIR-$(date +%F)
+  echo "$0: renaming $DESTDIR -> $NEWDESTDIR" | tee -a $logfile
+  mv $DESTDIR $NEWDESTDIR
+  cd $NEWDESTDIR || exit 3
+  sha256sum *.tar.xz *.zip *.exe > sha256sums.txt
+  gpg -a --clearsign --local-user 0x984496E7 sha256sums.txt || exit 2
+  cd ..
+  D=$(basename $NEWDESTDIR)
+  tar cf - $D/sha256sums* $D/*.tar.xz $D/*.zip $D/*.exe | ssh -i $PUBLISH_KEY $PUBLISH_HOST | tee -a $logfile
+else
+  echo "$0: giving up after $n tries" | tee -a $logfile
+fi
diff --git a/tools/continuous-builds/checkout-tbb.sh b/tools/continuous-builds/checkout-tbb.sh
new file mode 100755
index 0000000..f8a7929
--- /dev/null
+++ b/tools/continuous-builds/checkout-tbb.sh
@@ -0,0 +1,8 @@
+#! /bin/sh
+
+BUILDDIR=$1
+[ -z "$BUILDDIR" ] && BUILDDIR=~/usr/src/tor-browser-bundle/gitian
+
+cd $BUILDDIR || exit 1
+git checkout master
+git pull
diff --git a/tools/continuous-builds/park-nightly.sh b/tools/continuous-builds/park-nightly.sh
new file mode 100755
index 0000000..1ac5b2c
--- /dev/null
+++ b/tools/continuous-builds/park-nightly.sh
@@ -0,0 +1,42 @@
+#! /bin/sh                                                                      
+# usage: park-nightly.sh [DIR]
+
+V=1	#debug is enabled by default for now
+
+if [ "$1" = "-v" ]; then
+  V=1
+  shift
+fi
+
+DIR=$1
+[ -z "$DIR" ] && DIR=~/public_html/builds
+
+DSTDIR=tbb-nightly-$(date +%F)
+[ -z "$V" ] || echo "Aiming to fill up $DSTDIR"
+
+do_check() {
+    [ -z "$1" ] || cd $1 || exit 5
+    [ -z "$V" ] || echo "Verifying sha256sums.txt"
+    gpg -q --verify sha256sums.txt.asc > /dev/null || exit 3
+    [ -z "$V" ] || echo "Checking sha256sums.txt"
+    sha256sum --strict --quiet -c sha256sums.txt || exit 4
+}
+
+if [ -d $DIR/$DSTDIR ] && [ -e $DIR/$DSTDIR/tbb-nightly.stamp ]; then
+    [ -z "$V" ] || echo "Files already here, just doing the checking"
+    do_check $DIR/$DSTDIR
+    exit
+fi
+
+[ -d .staging ] || mkdir .staging
+chmod 700 .staging; cd .staging
+[ -z "$V" ] || echo "Saving files to disk"
+TAROPT=x
+[ -z "$V" ] || TAROPT=${TAROPT}v
+tar $TAROPT -f - || exit 6
+touch $DSTDIR/tbb-nightly.stamp
+
+do_check $DSTDIR || exit 2
+[ -d $DIR/$DSTDIR ] && [ -e $DIR/$DSTDIR/tbb-nightly.stamp ] && rm -rf $DIR/$DSTDIR
+cd ..; mv $DSTDIR $DIR/ || exit 1
+[ -z "$V" ] || echo "All good, all good"
diff --git a/tools/continuous-builds/setup-gitian-build-env.sh b/tools/continuous-builds/setup-gitian-build-env.sh
new file mode 100644
index 0000000..e9f435a
--- /dev/null
+++ b/tools/continuous-builds/setup-gitian-build-env.sh
@@ -0,0 +1,3 @@
+# source me!
+export LC_ALL=C
+export NUM_PROCS=10