diff --git a/mod_ci/cron.py b/mod_ci/cron.py index 7babb232..a46fdf6b 100755 --- a/mod_ci/cron.py +++ b/mod_ci/cron.py @@ -1,12 +1,16 @@ #!/usr/bin/python3 """handle cron logic for CI platform.""" +import fcntl import sys +import tempfile from os import path # Need to append server root path to ensure we can import the necessary files. sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) +LOCK_FILE = path.join(tempfile.gettempdir(), 'sample-platform-cron.lock') + def cron(testing=False): """Script to run from cron for Sampleplatform.""" @@ -17,6 +21,17 @@ def cron(testing=False): from mod_ci.controllers import TestPlatform, gcp_instance, start_platforms from run import config, log + # Cron starts a run every 10 minutes, but a run working through a backlog can take longer. + # Overlapping runs ran the server out of memory, so only one runs at a time. The lock is + # released when the process exits, even if it is killed. + lock = open(LOCK_FILE, 'w') + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + log.warning('Previous cron run is still in progress, skipping this one.') + lock.close() + return + log.info('Run the cron for kicking off CI platform(s).') github_token = config['GITHUB_TOKEN'] if not github_token: diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index 252e5c5c..a9a69bab 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -238,6 +238,19 @@ def test_cron_job_testing_true(self, mock_gcp_instance, mock_get_compute_service mock_delete_expired_instances.assert_not_called() mock_get_compute_service_object.assert_not_called() + @mock.patch('mod_ci.controllers.start_platforms') + @mock.patch('run.log') + def test_cron_job_skips_while_previous_run_holds_lock(self, mock_log, mock_start_platforms): + """Test cron does nothing while a previous cron run still holds the lock.""" + import fcntl + + from mod_ci.cron import LOCK_FILE, cron + with open(LOCK_FILE, 'w') as held: + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + cron() + mock_start_platforms.assert_not_called() + mock_log.warning.assert_called_once_with('Previous cron run is still in progress, skipping this one.') + @mock.patch.dict('run.config', {'GITHUB_TOKEN': '', 'GITHUB_OWNER': 'test', 'GITHUB_REPOSITORY': 'test'}) @mock.patch('run.log') def test_cron_job_empty_token(self, mock_log):