diff --git a/CHANGES.rst b/CHANGES.rst index b5dde2150a..dc7ed5c9cd 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,6 +26,12 @@ Common Compute ~~~~~~~ +- [GCE] Preserve completed results and pending node names on the exception + raised when bulk node creation times out, allowing callers to recover or + clean up resources without losing track of them. + (GITHUB-1589) + [Ishu Patel - @ishu7w] + - [OpenStack] Select service catalog endpoints by service type without filtering by the default service name. The service name is only used as a filter when explicitly provided via ``ex_force_service_name``. diff --git a/libcloud/compute/drivers/gce.py b/libcloud/compute/drivers/gce.py index e9d77c6e3a..a22704f49d 100644 --- a/libcloud/compute/drivers/gce.py +++ b/libcloud/compute/drivers/gce.py @@ -649,6 +649,25 @@ def __repr__(self): return ''.format(self.name, self.code) +class GCEMultipleNodesTimeoutError(LibcloudError): + """ + Node creation timed out with some results still pending. + + ``nodes`` contains the known results, including :class:`GCEFailedNode` + objects when errors were ignored. ``pending_node_names`` identifies nodes + whose final state is unknown; they may finish creating after the timeout. + No resources are automatically deleted. + """ + + def __init__(self, timeout, nodes, pending_node_names, driver): + self.nodes = nodes + self.pending_node_names = pending_node_names + super().__init__( + "Timeout (%s sec) while waiting for multiple instances" % timeout, + driver=driver, + ) + + class GCEHealthCheck(UuidMixin): """A GCE Http Health Check class.""" @@ -5680,7 +5699,11 @@ def ex_create_multiple_nodes( :type ex_service_accounts: ``list`` :keyword timeout: The number of seconds to wait for all nodes to be - created before timing out. + created before timing out. On timeout, + :class:`GCEMultipleNodesTimeoutError` exposes known + results in ``nodes`` and unresolved names in + ``pending_node_names``. Resources are not deleted; + pending operations may still complete. :type timeout: ``int`` :keyword description: The description of the node (instance). @@ -5817,7 +5840,14 @@ def ex_create_multiple_nodes( while not complete: if time.time() - start_time >= timeout: - raise Exception("Timeout (%s sec) while waiting for multiple " "instances") + raise GCEMultipleNodesTimeoutError( + timeout=timeout, + nodes=[status["node"] for status in status_list if status["node"] is not None], + pending_node_names=[ + status["name"] for status in status_list if status["node"] is None + ], + driver=self, + ) complete = True time.sleep(poll_interval) diff --git a/libcloud/test/compute/test_gce.py b/libcloud/test/compute/test_gce.py index f64fe8ad61..2dc926a6c6 100644 --- a/libcloud/test/compute/test_gce.py +++ b/libcloud/test/compute/test_gce.py @@ -46,12 +46,14 @@ GCEProject, GCEFirewall, GCENodeImage, + GCEFailedNode, GCENodeDriver, GCESubnetwork, GCEHealthCheck, GCEBackendService, GCEForwardingRule, GCETargetHttpProxy, + GCEMultipleNodesTimeoutError, timestamp_to_datetime, ) from libcloud.test.common.test_google import GoogleTestCase, GoogleAuthMockHttp @@ -1648,6 +1650,47 @@ def test_ex_create_multiple_nodes_image_family(self): ex_image_family="coreos-stable", ) + def test_ex_create_multiple_nodes_timeout_preserves_progress(self): + image = self.driver.ex_get_image("debian-7") + size = self.driver.ex_get_size("n1-standard-1") + created = Node("node-id", "lcnode-000", NodeState.RUNNING, [], [], self.driver) + failed = GCEFailedNode("lcnode-001", "quota exceeded", "QUOTA_EXCEEDED") + + def create_node(status, node_attrs): + if status["name"] == created.name: + status["node"] = created + elif status["name"] == failed.name: + status["node"] = failed + else: + status["node_response"] = {"selfLink": "pending-operation"} + + with ( + mock.patch.object(self.driver, "_multi_create_node", side_effect=create_node), + mock.patch("libcloud.compute.drivers.gce.time") as clock, + mock.patch.object(self.driver, "destroy_node") as destroy, + ): + clock.time.side_effect = [0, 0, 5] + with self.assertRaises(GCEMultipleNodesTimeoutError) as raised: + self.driver.ex_create_multiple_nodes( + "lcnode", size, image, 3, timeout=5, poll_interval=0 + ) + error = raised.exception + self.assertEqual(error.nodes, [created, failed]) + self.assertEqual(error.pending_node_names, ["lcnode-002"]) + self.assertIn("5 sec", str(error)) + self.assertIs(error.driver, self.driver) + destroy.assert_not_called() + + def test_ex_create_multiple_nodes_timeout_before_start(self): + image = self.driver.ex_get_image("debian-7") + size = self.driver.ex_get_size("n1-standard-1") + with mock.patch.object(self.driver, "_multi_create_node") as create: + with self.assertRaises(GCEMultipleNodesTimeoutError) as raised: + self.driver.ex_create_multiple_nodes("lcnode", size, image, 2, timeout=0) + self.assertEqual(raised.exception.nodes, []) + self.assertEqual(raised.exception.pending_node_names, ["lcnode-000", "lcnode-001"]) + create.assert_not_called() + def test_ex_create_targethttpproxy(self): proxy_name = "web-proxy" urlmap_name = "web-map"