Handle case when instances don't have all EC2 info yet

Handle corner case when instance is just launched and the describe doesn't report yet all the EC2 info like the private IP address.
In this case, the get_cluster_instances must not completely fail, but only skip the instance for which there aren't all the info yet.

The get_cluster_instances is used in the compute fleet stopped loop to keep the nodes to down and in the manage cluster loop to update slurm nodes with EC2 info and to terminate orphaned nodes.

Signed-off-by: Luca Carrogu <[email protected]>

Author: lukeseawalker

src/slurm_plugin/instance_manager.py

@@ -342,7 +342,11 @@ def get_unhealthy_cluster_instance_status(self, cluster_instance_ids):
 
     @log_exception(logger, "getting cluster instances from EC2", raise_on_error=True)
     def get_cluster_instances(self, include_head_node=False, alive_states_only=True):
-        """Get instances that are associated with the cluster."""
+        """
+        Get instances that are associated with the cluster.
+
+        Instances without all the info set are ignored and not returned
+        """
         ec2_client = boto3.client("ec2", region_name=self._region, config=self._boto3_config)
         paginator = ec2_client.get_paginator("describe_instances")
         args = {
@@ -354,15 +358,27 @@ def get_cluster_instances(self, include_head_node=False, alive_states_only=True)
             args["Filters"].append({"Name": "tag:parallelcluster:node-type", "Values": ["Compute"]})
         response_iterator = paginator.paginate(PaginationConfig={"PageSize": BOTO3_PAGINATION_PAGE_SIZE}, **args)
         filtered_iterator = response_iterator.search("Reservations[].Instances[]")
-        return [
-            EC2Instance(
-                instance_info["InstanceId"],
-                instance_info["PrivateIpAddress"],
-                instance_info["PrivateDnsName"].split(".")[0],
-                instance_info["LaunchTime"],
-            )
-            for instance_info in filtered_iterator
-        ]
+
+        instances = []
+        for instance_info in filtered_iterator:
+            try:
+                instances.append(
+                    EC2Instance(
+                        instance_info["InstanceId"],
+                        instance_info["PrivateIpAddress"],
+                        instance_info["PrivateDnsName"].split(".")[0],
+                        instance_info["LaunchTime"],
+                    )
+                )
+            except Exception as e:
+                logger.warning(
+                    "Ignoring instance %s because not all EC2 info are available, exception: %s, message: %s",
+                    instance_info["InstanceId"],
+                    type(e).__name__,
+                    e,
+                )
+
+        return instances
 
     def terminate_all_compute_nodes(self, terminate_batch_size):
         try:

tests/slurm_plugin/slurm_resources/test_instance_manager.py

@@ -1514,7 +1514,7 @@ def get_unhealthy_cluster_instance_status(
     @pytest.mark.parametrize(
         "mock_kwargs, mocked_boto3_request, expected_parsed_result",
         [
-            (
+            pytest.param(
                 {"include_head_node": False, "alive_states_only": True},
                 MockedBoto3Request(
                     method="describe_instances",
@@ -1552,8 +1552,9 @@ def get_unhealthy_cluster_instance_status(
                     EC2Instance("i-1", "ip-1", "hostname", datetime(2020, 1, 1, tzinfo=timezone.utc)),
                     EC2Instance("i-2", "ip-2", "hostname", datetime(2020, 1, 1, tzinfo=timezone.utc)),
                 ],
+                id="default",
             ),
-            (
+            pytest.param(
                 {"include_head_node": False, "alive_states_only": True},
                 MockedBoto3Request(
                     method="describe_instances",
@@ -1569,8 +1570,9 @@ def get_unhealthy_cluster_instance_status(
                     generate_error=False,
                 ),
                 [],
+                id="empty_response",
             ),
-            (
+            pytest.param(
                 {"include_head_node": True, "alive_states_only": False},
                 MockedBoto3Request(
                     method="describe_instances",
@@ -1595,9 +1597,46 @@ def get_unhealthy_cluster_instance_status(
                     generate_error=False,
                 ),
                 [EC2Instance("i-1", "ip-1", "hostname", datetime(2020, 1, 1, tzinfo=timezone.utc))],
+                id="custom_args",
+            ),
+            pytest.param(
+                {"include_head_node": False, "alive_states_only": True},
+                MockedBoto3Request(
+                    method="describe_instances",
+                    response={
+                        "Reservations": [
+                            {
+                                "Instances": [
+                                    {
+                                        "InstanceId": "i-1",
+                                        "LaunchTime": datetime(2020, 1, 1, tzinfo=timezone.utc),
+                                    },
+                                    {
+                                        "InstanceId": "i-2",
+                                        "PrivateIpAddress": "ip-2",
+                                        "PrivateDnsName": "hostname",
+                                        "LaunchTime": datetime(2020, 1, 1, tzinfo=timezone.utc),
+                                    },
+                                ]
+                            }
+                        ]
+                    },
+                    expected_params={
+                        "Filters": [
+                            {"Name": "tag:parallelcluster:cluster-name", "Values": ["hit"]},
+                            {"Name": "instance-state-name", "Values": list(EC2_INSTANCE_ALIVE_STATES)},
+                            {"Name": "tag:parallelcluster:node-type", "Values": ["Compute"]},
+                        ],
+                        "MaxResults": 1000,
+                    },
+                    generate_error=False,
+                ),
+                [
+                    EC2Instance("i-2", "ip-2", "hostname", datetime(2020, 1, 1, tzinfo=timezone.utc)),
+                ],
+                id="no_ec2_info",
             ),
         ],
-        ids=["default", "empty_response", "custom_args"],
     )
     def test_get_cluster_instances(
         self, mock_kwargs, mocked_boto3_request, expected_parsed_result, instance_manager, boto3_stubber, mocker