[tbb-commits] [tor-browser-bundle/master] Bug 14959: Download missing MAR files for incrementals

[mikeperry at torproject.org]

commit 803049749f966e63e7c4f86dd556c49fa0fdd48d
Author: Nicolas Vigier <boklm at torproject.org>
Date:   Mon Mar 16 18:49:43 2015 +0100

    Bug 14959: Download missing MAR files for incrementals
---
 gitian/Makefile                                  |    6 ++-
 gitian/gpg/torbrowser.gpg                        |  Bin 0 -> 6015 bytes
 tools/update-responses/config.yml                |    3 ++
 tools/update-responses/download_missing_versions |    1 +
 tools/update-responses/update_responses          |   58 +++++++++++++++++++++-
 5 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/gitian/Makefile b/gitian/Makefile
index ea82151..8f89bca 100644
--- a/gitian/Makefile
+++ b/gitian/Makefile
@@ -29,10 +29,12 @@ build-alpha:
 	./mkbundle-mac.sh versions.alpha
 
 incrementals:
-	../tools/update-responses/gen_incrementals release || echo 'Warning: could not generate incremental MARs.' >&2
+	$(TORSOCKS) ../tools/update-responses/download_missing_versions release
+	../tools/update-responses/gen_incrementals release
 
 incrementals-alpha:
-	../tools/update-responses/gen_incrementals alpha || echo 'Warning: could not generate incremental MARs.' >&2
+	$(TORSOCKS) ../tools/update-responses/download_missing_versions alpha
+	../tools/update-responses/gen_incrementals alpha
 
 signmars:
 	./signmars.sh versions
diff --git a/gitian/gpg/torbrowser.gpg b/gitian/gpg/torbrowser.gpg
new file mode 100644
index 0000000..7267d4d
Binary files /dev/null and b/gitian/gpg/torbrowser.gpg differ
diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml
index a338745..403be5c 100644
--- a/tools/update-responses/config.yml
+++ b/tools/update-responses/config.yml
@@ -1,4 +1,7 @@
 ---
+download:
+    archive_url: https://archive.torproject.org/tor-package-archive/torbrowser
+    gpg_keyring: ../../gitian/gpg/torbrowser.gpg
 build_targets:
     linux32: Linux_x86-gcc3
     linux64: Linux_x86_64-gcc3
diff --git a/tools/update-responses/download_missing_versions b/tools/update-responses/download_missing_versions
new file mode 120000
index 0000000..3766925
--- /dev/null
+++ b/tools/update-responses/download_missing_versions
@@ -0,0 +1 @@
+update_responses
\ No newline at end of file
diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses
index 8b8c217..07efc7c 100755
--- a/tools/update-responses/update_responses
+++ b/tools/update-responses/update_responses
@@ -6,9 +6,10 @@ use English;
 use FindBin;
 use YAML qw(LoadFile);
 use File::Slurp;
-use Digest::SHA;
+use Digest::SHA qw(sha256_hex);
 use XML::Writer;
 use Cwd;
+use File::Copy;
 use File::Temp;
 use File::Find;
 use File::Which;
@@ -408,6 +409,55 @@ sub check_update_responses_channel {
     }
 }
 
+sub download_version {
+    my ($config, $version) = @_;
+    my $tmpdir = File::Temp->newdir();
+    my $destdir = "$releases_dir/$version";
+    my $urldir = "$config->{download}{archive_url}/$version";
+    print "Downloading version $version\n";
+    foreach my $file (qw(sha256sums.txt sha256sums.txt.asc)) {
+        if (getstore("$urldir/$file", "$tmpdir/$file") != 200) {
+            exit_error "Error downloading $urldir/$file";
+        }
+    }
+    if (system('gpg', '--no-default-keyring', '--keyring',
+            $config->{download}{gpg_keyring}, '--verify',
+            "$tmpdir/sha256sums.txt.asc", "$tmpdir/sha256sums.txt")) {
+        exit_error "Error checking gpg signature for version $version";
+    }
+    mkdir $destdir;
+    move "$tmpdir/sha256sums.txt.asc", "$destdir/sha256sums.txt.asc";
+    move "$tmpdir/sha256sums.txt", "$destdir/sha256sums.txt";
+    my %sums = map { chomp; reverse split '  ', $_ } read_file "$destdir/sha256sums.txt";
+    foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) {
+        print "Downloading $file\n";
+        exit_error "Error downloading $urldir/$file\n"
+                unless getstore("$urldir/$file", "$tmpdir/$file") == 200;
+        if ($sums{$file} ne sha256_hex(read_file("$tmpdir/$file"))) {
+            exit_error "Error unsigning $file"
+                if system('signmar', '-r', "$tmpdir/$file", "$tmpdir/$file.u");
+            exit_error "Wrong checksum for $file"
+                unless $sums{$file} eq sha256_hex(read_file("$tmpdir/$file.u"));
+            move "$tmpdir/$file.u", "$tmpdir/$file";
+        }
+        move "$tmpdir/$file", "$destdir/$file";
+    }
+}
+
+sub download_missing_versions {
+    my ($config, @channels) = @_;
+    foreach my $channel (@channels) {
+        exit_error "Unknown channel $channel"
+                unless $config->{channels}{$channel};
+        my $cversion = $config->{channels}{$channel};
+        next unless $config->{versions}{$cversion}{incremental_from};
+        foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) {
+            next if -d "$releases_dir/$version";
+            download_version($config, $version);
+        }
+    }
+}
+
 sub check_update_responses {
     my ($config) = @_;
     exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV;
@@ -453,6 +503,12 @@ my %actions = (
             create_incremental_mars_for_version($config, $version);
         }
     },
+    download_missing_versions => sub {
+        my ($config) = @_;
+        my @channels = @ARGV ? @ARGV : keys %{$config->{channels}};
+        extract_martools;
+        download_missing_versions($config, @channels);
+    },
     check_update_responses_deployement => \&check_update_responses,
 );