Add mirror-all-snapshots.py script for extra crowdsourced backup power.

3 files changed, 43 insertions, 5 deletions

diff --git a/src/etc/get-snapshot.py b/src/etc/get-snapshot.py
index 806d37511dd..98e480defda 100755
--- a/src/etc/get-snapshot.py
+++ b/src/etc/get-snapshot.py
@@ -3,11 +3,6 @@
 import os, tarfile, hashlib, re, shutil
 from snapshot import *
 
-def snap_filename_hash_part(snap):
-  match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
-  if not match:
-    raise Exception("unable to find hash in filename: " + snap)
-  return match.group(1)
 
 def unpack_snapshot(snap):
   dl_path = os.path.join(download_dir_base, snap)
diff --git a/src/etc/mirror-all-snapshots.py b/src/etc/mirror-all-snapshots.py
new file mode 100644
index 00000000000..f4bb8c042ea
--- /dev/null
+++ b/src/etc/mirror-all-snapshots.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+
+import os, tarfile, hashlib, re, shutil
+from snapshot import *
+
+f = open(snapshotfile)
+date = None
+rev = None
+platform = None
+snap = None
+i = 0
+
+for line in f.readlines():
+    i += 1
+    parsed = parse_line(i, line)
+    if (not parsed): continue
+
+    if parsed["type"] == "snapshot":
+        date = parsed["date"]
+        rev = parsed["rev"]
+
+    elif rev != None and parsed["type"] == "file":
+        platform = parsed["platform"]
+        hsh = parsed["hash"]
+        snap = full_snapshot_name(date, rev, platform, hsh)
+        dl = os.path.join(download_dir_base, snap)
+        url = download_url_base + "/" + snap
+        if (not os.path.exists(dl)):
+            print("downloading " + url)
+            get_url_to_file(url, dl)
+        if (snap_filename_hash_part(snap) == hash_file(dl)):
+            print("got download with ok hash")
+        else:
+            raise Exception("bad hash on download")
+
+
+
diff --git a/src/etc/snapshot.py b/src/etc/snapshot.py
index 898f4f79e5f..26015a5427f 100644
--- a/src/etc/snapshot.py
+++ b/src/etc/snapshot.py
@@ -95,6 +95,12 @@ def local_rev_committer_date():
 def get_url_to_file(u,f):
   subprocess.check_call(["curl", "-o", f, u])
 
+def snap_filename_hash_part(snap):
+  match = re.match(r".*([a-fA-F\d]{40}).tar.bz2$", snap)
+  if not match:
+    raise Exception("unable to find hash in filename: " + snap)
+  return match.group(1)
+
 def hash_file(x):
     h = hashlib.sha1()
     h.update(open(x, "rb").read())