diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index fc55c8a9..59128473 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -719,7 +719,12 @@ def gcp_instance(app, db, platform, repository, delay) -> None: except Exception as e: log.error(f"Unexpected error checking PR status for test {test.id}: {e}") continue - start_test(compute, app, db, repository, test, github_config['bot_token']) + try: + start_test(compute, app, db, repository, test, github_config['bot_token']) + except Exception: + # One test that can't be started (e.g. its files under vm_data raising OSError) must not + # stop the tests queued after it, or the other platform. It stays pending and is retried. + log.exception(f"[{platform}] Test {test.id}: could not be started, will retry on next cron run") def get_compute_service_object() -> googleapiclient.discovery.Resource: diff --git a/tests/test_ci/test_controllers.py b/tests/test_ci/test_controllers.py index c1867685..252e5c5c 100644 --- a/tests/test_ci/test_controllers.py +++ b/tests/test_ci/test_controllers.py @@ -171,6 +171,27 @@ def test_start_platform_windows_specified(self, mock_log, mock_gcp_instance, mock_delete_expired_instances.assert_called_once() mock_get_compute_service_object.assert_called_once() + @mock.patch('mod_ci.controllers.start_test') + @mock.patch('mod_ci.controllers.get_compute_service_object') + @mock.patch('mod_ci.controllers.delete_expired_instances') + @mock.patch('run.log') + def test_start_platforms_continues_after_a_test_fails_to_start(self, mock_log, mock_delete_expired_instances, + mock_get_compute_service_object, mock_start_test): + """Test that a test failing to start doesn't stop the tests after it or the other platform.""" + commit = "1978060bf7d2edd119736ba3ba88341f3bec3323" + queued = [Test(TestPlatform.linux, TestType.commit, 1, "master", commit), + Test(TestPlatform.linux, TestType.commit, 1, "master", commit), + Test(TestPlatform.windows, TestType.commit, 1, "master", commit)] + g.db.add_all(queued) + g.db.commit() + queued_ids = [test.id for test in queued] + mock_start_test.side_effect = [OSError(5, 'Input/output error'), None, None] + + start_platforms(mock.ANY) + + self.assertEqual(queued_ids, [call.args[4].id for call in mock_start_test.call_args_list]) + mock_log.exception.assert_called_once() + @mock.patch('run.log') @mock.patch('mod_ci.controllers.MaintenanceMode') def test_gcp_instance_maintenance_mode(self, mock_maintenance, mock_log):