Lock x.py build state

Prevent spurious build failures and other bugs caused by parallel runs of
x.py. We back the lock with sqlite, so that we have a cross-platform locking
strategy, and which can be done nearly first in the build process (from Python),
which helps move the lock as early as possible.

1 files changed, 42 insertions, 0 deletions

diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py
index 22f2e405a1e..83ba58bc7bd 100644
--- a/src/bootstrap/bootstrap.py
+++ b/src/bootstrap/bootstrap.py
@@ -15,6 +15,42 @@ import tempfile
 
 from time import time
 
+# Acquire a lock on the build directory to make sure that
+# we don't cause a race condition while building
+# Lock is created in `build_dir/lock.db`
+def acquire_lock(build_dir):
+    try:
+        import sqlite3
+
+        path = os.path.join(build_dir, "lock.db")
+        try:
+            con = sqlite3.Connection(path, timeout=0)
+            curs = con.cursor()
+            curs.execute("BEGIN EXCLUSIVE")
+            # The lock is released when the cursor is dropped
+            return curs
+        # If the database is busy then lock has already been acquired
+        # so we wait for the lock.
+        # We retry every quarter second so that execution is passed back to python
+        # so that it can handle signals
+        except sqlite3.OperationalError:
+            del con
+            del curs
+            print("Waiting for lock on build directory")
+            con = sqlite3.Connection(path, timeout=0.25)
+            curs = con.cursor()
+            while True:
+                try:
+                    curs.execute("BEGIN EXCLUSIVE")
+                except sqlite3.OperationalError:
+                    pass
+            return curs
+    except ImportError:
+        print("warning: sqlite3 not available in python, skipping build directory lock")
+        print("please file an issue on rust-lang/rust")
+        print("this is not a problem for non-concurrent x.py invocations")
+        return None
+
 def support_xz():
     try:
         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -1225,6 +1261,12 @@ def bootstrap(help_triggered):
     build.set_dist_environment(data["dist_server"])
 
     build.build = args.build or build.build_triple()
+
+    # Acquire the lock before doing any build actions
+    # The lock is released when `lock` is dropped
+    if not os.path.exists(build.build_dir):
+        os.makedirs(build.build_dir)
+    lock = acquire_lock(build.build_dir)
     build.update_submodules()
 
     # Fetch/build the bootstrap