[FIX] CI cron: skip a run while the previous one is still running

mod_ci/cron.py

@@ -1,12 +1,16 @@
 #!/usr/bin/python3
 """handle cron logic for CI platform."""
 
+import fcntl
 import sys
+import tempfile
 from os import path
 
 # Need to append server root path to ensure we can import the necessary files.
 sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
 
+LOCK_FILE = path.join(tempfile.gettempdir(), 'sample-platform-cron.lock')
+
 
 def cron(testing=False):
     """Script to run from cron for Sampleplatform."""
@@ -17,6 +21,17 @@ def cron(testing=False):
     from mod_ci.controllers import TestPlatform, gcp_instance, start_platforms
     from run import config, log
 
+    # Cron starts a run every 10 minutes, but a run working through a backlog can take longer.
+    # Overlapping runs ran the server out of memory, so only one runs at a time. The lock is
+    # released when the process exits, even if it is killed.
+    lock = open(LOCK_FILE, 'w')
+    try:
+        fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except BlockingIOError:
+        log.warning('Previous cron run is still in progress, skipping this one.')
+        lock.close()
+        return
+
     log.info('Run the cron for kicking off CI platform(s).')
     github_token = config['GITHUB_TOKEN']
     if not github_token:

tests/test_ci/test_controllers.py

@@ -238,6 +238,19 @@ def test_cron_job_testing_true(self, mock_gcp_instance, mock_get_compute_service
         mock_delete_expired_instances.assert_not_called()
         mock_get_compute_service_object.assert_not_called()
 
+    @mock.patch('mod_ci.controllers.start_platforms')
+    @mock.patch('run.log')
+    def test_cron_job_skips_while_previous_run_holds_lock(self, mock_log, mock_start_platforms):
+        """Test cron does nothing while a previous cron run still holds the lock."""
+        import fcntl
+
+        from mod_ci.cron import LOCK_FILE, cron
+        with open(LOCK_FILE, 'w') as held:
+            fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            cron()
+        mock_start_platforms.assert_not_called()
+        mock_log.warning.assert_called_once_with('Previous cron run is still in progress, skipping this one.')
+
     @mock.patch.dict('run.config', {'GITHUB_TOKEN': '', 'GITHUB_OWNER': 'test', 'GITHUB_REPOSITORY': 'test'})
     @mock.patch('run.log')
     def test_cron_job_empty_token(self, mock_log):