ENH: Add rule to remove downtime

lib/enums/hypervisor_states.py

@@ -30,6 +30,19 @@ class HypervisorState(Enum):
         "state": True,
         "servers": False,
     }
+    REBOOTED = {
+        "uptime": True,
+        "enabled": False,
+        "state": True,
+        "servers": False,
+    }
+    EMPTY = {
+        "uptime": True,
+        "enabled": True,
+        "state": True,
+        "servers": False,
+    }
+    DOWN = auto()
     UNKNOWN = auto()
 
     @classmethod

lib/openstack_api/openstack_hypervisor.py

@@ -9,9 +9,10 @@ def get_hypervisor_state(hypervisor: Dict, uptime_limit: int) -> HypervisorState
     :param uptime_limit: Number of days of uptime before hypervisor requires maintenance
     :return: Hypervisor state
     """
+    if hypervisor["hypervisor_state"] == "down":
+        return HypervisorState.DOWN
     if not valid_state(hypervisor):
         return HypervisorState.UNKNOWN
-
     hv_state = {
         "uptime": hypervisor["hypervisor_uptime_days"] < uptime_limit,
         "enabled": hypervisor["hypervisor_status"] == "enabled",

lib/workflows/hv_patch_and_reboot.py

@@ -1,6 +1,6 @@
 import datetime
-
-from icinga_api.downtime import schedule_downtime, remove_downtime
+from enums.icinga.icinga_objects import IcingaObject
+from icinga_api.downtime import schedule_downtime
 from structs.icinga.downtime_details import DowntimeDetails
 from structs.icinga.icinga_account import IcingaAccount
 from structs.ssh.ssh_connection_details import SSHDetails
@@ -28,7 +28,7 @@ def patch_and_reboot(
     start_timestamp = int(start_time.timestamp())
     end_timestamp = int(end_time.timestamp())
     downtime_details = DowntimeDetails(
-        object_type="Host",
+        object_type=IcingaObject.HOST,
         object_name=hypervisor_name,
         start_time=start_timestamp,
         end_time=end_timestamp,
@@ -37,14 +37,6 @@ def patch_and_reboot(
         duration=end_timestamp - start_timestamp,
     )
     schedule_downtime(icinga_account=icinga_account, details=downtime_details)
-    try:
-        patch_out = ssh_client.run_command_on_host("patch")
-        reboot_out = ssh_client.run_command_on_host("reboot")
-
-    finally:
-        remove_downtime(
-            icinga_account=icinga_account,
-            object_type="Host",
-            object_name=hypervisor_name,
-        )
+    patch_out = ssh_client.run_command_on_host("patch")
+    reboot_out = ssh_client.run_command_on_host("reboot")
     return {"patch_output": patch_out.decode(), "reboot_output": reboot_out.decode()}

lib/workflows/icinga_downtime.py

@@ -3,6 +3,7 @@
 
 import pytz
 
+from enums.icinga.icinga_objects import IcingaObject
 from structs.icinga.downtime_details import DowntimeDetails
 from structs.icinga.icinga_account import IcingaAccount
 
@@ -34,7 +35,7 @@ def schedule_downtime(
     end_timestamp = int(utc_end_time.timestamp())
 
     downtime_details = DowntimeDetails(
-        object_type=object_type,
+        object_type=IcingaObject[object_type.upper()],
         object_name=name,
         start_time=start_timestamp,
         end_time=end_timestamp,
@@ -50,6 +51,6 @@ def remove_downtime(icinga_account: IcingaAccount, object_type: str, name: str):
 
     downtime.remove_downtime(
         icinga_account=icinga_account,
-        object_type=object_type,
+        object_type=IcingaObject[object_type.upper()],
         object_name=name,
     )

rules/hv.remove.downtime.yaml

@@ -0,0 +1,23 @@
+---
+name: "hv.remove.downtime"
+pack: "stackstorm_openstack"
+description: "Removes the downtime on a HV once it has successfullly rebooted"
+enabled: true
+
+criteria:
+  trigger.previous_state:
+    type: equals
+    pattern: DOWN
+  trigger.current_state:
+    type: equals
+    pattern: REBOOTED
+
+trigger:
+  type: "stackstorm_openstack.hypervisor.state_change"
+
+action:
+  ref: "stackstorm_openstack.icinga.remove.downtime"
+  parameters:
+    icinga_account_name: "default"
+    name: "{{ trigger.hypervisor_name }}"
+    object_type: "Host"

sensors/hypervisor.state_change.yaml

@@ -2,7 +2,7 @@
 class_name: HypervisorStateSensor
 entry_point: src/hypervisor_state_sensor.py
 description: Monitor state of Hypervisors
-poll_interval: 600
+poll_interval: 30
 trigger_types:
   - name: "hypervisor.state_change"
     description: "Triggers when the state of the hypervisor changes"

tests/lib/openstack_api/test_openstack_hypervisor.py

@@ -1,5 +1,6 @@
 from enums.hypervisor_states import HypervisorState
 from openstack_api.openstack_hypervisor import get_hypervisor_state
+import pytest
 
 
 def test_get_state_running():
@@ -58,68 +59,115 @@ def test_get_state_drained():
     assert state == HypervisorState.DRAINED
 
 
-def test_get_state_unkown_status():
+def test_get_state_rebooted():
     """
-    Test hypervisor state is unknown when missing status
+    Test hypervisor state is rebooted for given variables
     """
     hypervisor = {
-        "hypervisor_uptime_days": 100,
-        "hypervisor_status": None,
+        "hypervisor_uptime_days": 0,
+        "hypervisor_status": "disabled",
         "hypervisor_state": "up",
         "hypervisor_server_count": 0,
     }
     state = get_hypervisor_state(hypervisor, 60)
-    assert state == HypervisorState.UNKNOWN
+    assert state == HypervisorState.REBOOTED
 
 
-def test_get_state_unkown_state():
+def test_get_state_empty():
     """
-    Test hypervisor state is unknown when missing state
+    Test hypervisor state is empty for given variables
     """
     hypervisor = {
-        "hypervisor_uptime_days": 100,
+        "hypervisor_uptime_days": 40,
         "hypervisor_status": "enabled",
-        "hypervisor_state": None,
+        "hypervisor_state": "up",
         "hypervisor_server_count": 0,
     }
     state = get_hypervisor_state(hypervisor, 60)
-    assert state == HypervisorState.UNKNOWN
-
-
-def test_get_state_unkown_uptime():
+    assert state == HypervisorState.EMPTY
+
+
+@pytest.mark.parametrize(
+    "hypervisor",
+    [
+        {
+            "hypervisor_uptime_days": 40,
+            "hypervisor_status": "enabled",
+            "hypervisor_state": "down",
+            "hypervisor_server_count": 0,
+        },
+        {
+            "hypervisor_uptime_days": 100,
+            "hypervisor_status": "disabled",
+            "hypervisor_state": "down",
+            "hypervisor_server_count": 0,
+        },
+        {
+            "hypervisor_uptime_days": 40,
+            "hypervisor_status": "enabled",
+            "hypervisor_state": "down",
+            "hypervisor_server_count": 2,
+        },
+    ],
+)
+def test_get_state_down(hypervisor):
+    """
+    Test hypervisor state is down for given variables
     """
-    Test hypervisor state is unknown when missing uptime
+    state = get_hypervisor_state(hypervisor, 60)
+    assert state == HypervisorState.DOWN
+
+
+@pytest.mark.parametrize(
+    "hypervisor",
+    [
+        {
+            "hypervisor_uptime_days": None,
+            "hypervisor_status": "enabled",
+            "hypervisor_state": "up",
+            "hypervisor_server_count": 0,
+        },
+        {
+            "hypervisor_uptime_days": 100,
+            "hypervisor_status": None,
+            "hypervisor_state": "up",
+            "hypervisor_server_count": 0,
+        },
+        {
+            "hypervisor_uptime_days": 40,
+            "hypervisor_status": "enabled",
+            "hypervisor_state": None,
+            "hypervisor_server_count": 2,
+        },
+        {
+            "hypervisor_uptime_days": 40,
+            "hypervisor_status": "enabled",
+            "hypervisor_state": "up",
+            "hypervisor_server_count": None,
+        },
+        {
+            "hypervisor_uptime_days": 5,
+            "hypervisor_status": "enabled",
+            "hypervisor_state": "up",
+            "hypervisor_server_count": -2,
+        },
+    ],
+)
+def test_get_unkown_status(hypervisor):
+    """
+    Test hypervisor state is unknown when missing parameters, when state is up
     """
-    hypervisor = {
-        "hypervisor_uptime_days": None,
-        "hypervisor_status": "enabled",
-        "hypervisor_state": "up",
-        "hypervisor_server_count": 0,
-    }
     state = get_hypervisor_state(hypervisor, 60)
     assert state == HypervisorState.UNKNOWN
 
 
-def test_get_state_unkown_server_count():
+def test_missing():
     """
-    Test hypervisor state is unknown when missing server count
+    Test hypervisor state is missing for a missing value in class
     """
-    hypervisor = {
-        "hypervisor_uptime_days": 100,
-        "hypervisor_status": "enabled",
-        "hypervisor_state": "up",
-        "hypervisor_server_count": None,
-    }
-    state = get_hypervisor_state(hypervisor, 60)
-    assert state == HypervisorState.UNKNOWN
-
-
-def test_get_state_no_matching_state():
-    hypervisor = {
-        "hypervisor_uptime_days": 60,
-        "hypervisor_status": "disabled",
-        "hypervisor_state": "down",
-        "hypervisor_server_count": 10,
+    mock_hv_state = {
+        "uptime": True,
+        "enabled": False,
+        "state": True,
     }
-    result = get_hypervisor_state(hypervisor, 60)
-    assert result == HypervisorState.UNKNOWN
+    assert HypervisorState(mock_hv_state) == HypervisorState.UNKNOWN

tests/lib/workflows/test_hv_patch_and_reboot.py

@@ -1,5 +1,6 @@
 import datetime
 from unittest.mock import MagicMock, patch
+from enums.icinga.icinga_objects import IcingaObject
 from structs.icinga.downtime_details import DowntimeDetails
 from structs.ssh.ssh_connection_details import SSHDetails
 from workflows.hv_patch_and_reboot import patch_and_reboot
@@ -8,9 +9,9 @@
 
 @patch("workflows.hv_patch_and_reboot.schedule_downtime")
 @patch("workflows.hv_patch_and_reboot.SSHConnection")
-@patch("workflows.hv_patch_and_reboot.remove_downtime")
 def test_successful_patch_and_reboot(
-    mock_remove_downtime, mock_ssh_conn, mock_schedule_downtime
+    mock_ssh_conn,
+    mock_schedule_downtime,
 ):
     """
     Test successful running of patch and reboot workflow
@@ -35,7 +36,7 @@ def test_successful_patch_and_reboot(
     mock_schedule_downtime.assert_called_once_with(
         icinga_account=icinga_account,
         details=DowntimeDetails(
-            object_type="Host",
+            object_type=IcingaObject.HOST,
             object_name=mock_hypervisor_name,
             start_time=mock_start_timestamp,
             end_time=mock_end_timestamp,
@@ -56,19 +57,10 @@ def test_successful_patch_and_reboot(
     assert result["patch_output"] == "Patch command output"
     assert result["reboot_output"] == "Reboot command output"
 
-    mock_remove_downtime.assert_called_once_with(
-        icinga_account=icinga_account,
-        object_type="Host",
-        object_name=mock_hypervisor_name,
-    )
-
 
 @patch("workflows.hv_patch_and_reboot.schedule_downtime")
 @patch("workflows.hv_patch_and_reboot.SSHConnection")
-@patch("workflows.hv_patch_and_reboot.remove_downtime")
-def test_failed_schedule_downtime(
-    mock_remove_downtime, mock_ssh_conn, mock_schedule_downtime
-):
+def test_failed_schedule_downtime(mock_ssh_conn, mock_schedule_downtime):
     """
     Test unsuccessful running of patch and reboot workflow - where the schedule
     downtime raises an exception.
@@ -78,7 +70,6 @@ def test_failed_schedule_downtime(
     mock_hypervisor_name = "test_host"
     mock_schedule_downtime.side_effect = Exception
 
-    # test that schedule downtime is called once with expected parameters
     with pytest.raises(Exception):
         patch_and_reboot(
             icinga_account,
@@ -96,13 +87,11 @@ def test_failed_schedule_downtime(
     )
 
     mock_ssh_conn.return_value.run_command_on_host.assert_not_called()
-    mock_remove_downtime.assert_not_called()
 
 
 @patch("workflows.hv_patch_and_reboot.schedule_downtime")
 @patch("workflows.hv_patch_and_reboot.SSHConnection")
-@patch("workflows.hv_patch_and_reboot.remove_downtime")
-def test_failed_ssh(mock_remove_downtime, mock_ssh_conn, mock_schedule_downtime):
+def test_failed_ssh(mock_ssh_conn, mock_schedule_downtime):
     """
     Test unsuccessful running of patch and reboot workflow - where either ssh command
     fails
@@ -133,7 +122,7 @@ def test_failed_ssh(mock_remove_downtime, mock_ssh_conn, mock_schedule_downtime)
     mock_schedule_downtime.assert_called_once_with(
         icinga_account=icinga_account,
         details=DowntimeDetails(
-            object_type="Host",
+            object_type=IcingaObject.HOST,
             object_name=mock_hypervisor_name,
             start_time=mock_start_timestamp,
             end_time=mock_end_timestamp,
@@ -142,8 +131,3 @@ def test_failed_ssh(mock_remove_downtime, mock_ssh_conn, mock_schedule_downtime)
             duration=mock_end_timestamp - mock_start_timestamp,
         ),
     )
-    mock_remove_downtime.assert_called_once_with(
-        icinga_account=icinga_account,
-        object_type="Host",
-        object_name=mock_hypervisor_name,
-    )