[Monit] Sanity check for Monit services (#2501)

yozhao101 · yxieca · web-flow · commit 7aeb49f2b80e · 2020-11-20T10:26:25.000-08:00
Description of PR

Summary:
This PR aims to add Monit service into sanity check. Specifically before running each pytest script, sanity check will determine whether the Monit is running and at the same time determine whether the services which were monitored by Monit are running correctly or not.

Fixes # (issue)

Type of change
[ x] Testbed and Framework(new/improvement)

Approach
What is the motivation for this PR?
This PR aims to add Monit service into sanity check. Specifically before running each pytest script, sanity check will determine whether the Monit is running and at the same time determine whether the services which were monitored by Monit are running correctly or not.

How did you do it?
I added a new function check_monit(...) in checks.py to achieve the purpose. In this function, it will first call a common function get_monit_services_status(...) in devices.py to get the metadata (service name, service status and service type) of services which are monitored by Monit. Then it will determine whether Monit is running or not and whether each service which was monitored by Monit is running or not.

How did you verify/test it?
I tested this change against the lab device str-dx010-acs-1.

Any platform specific information?
N/A

Supported testbed topology if it's a new test case?
N/A

Signed-off-by: Yong Zhao &lt;yozhao@microsoft.com&gt;

Co-authored-by: Ying Xie &lt;yxieca@users.noreply.github.com&gt;
diff --git a/tests/common/devices.py b/tests/common/devices.py
@@ -403,6 +403,34 @@ def critical_services_fully_started(self):
         logging.debug("Status of critical services: %s" % str(result))
         return all(result.values())
 
+    def get_monit_services_status(self):
+        """
+        @summary: Get metadata (service name, service status and service type) of services
+                  which were monitored by Monit.
+        @return: A dictionary in which key is the service name and values are service status
+                 and service type.
+        """
+        monit_services_status = {}
+
+        services_status_result = self.shell("sudo monit status", module_ignore_errors=True)
+
+        exit_code = services_status_result["rc"]
+        if exit_code != 0:
+            return monit_services_status
+
+        for index, service_info in enumerate(services_status_result["stdout_lines"]):
+            if "status" in service_info and "monitoring status" not in service_info:
+                service_type_name = services_status_result["stdout_lines"][index - 1]
+                service_type = service_type_name.split("'")[0].strip()
+                service_name = service_type_name.split("'")[1].strip()
+                service_status = service_info[service_info.find("status") + len("status"):].strip()
+
+                monit_services_status[service_name] = {}
+                monit_services_status[service_name]["service_status"] = service_status
+                monit_services_status[service_name]["service_type"] = service_type
+
+        return monit_services_status
+
     def get_critical_group_and_process_lists(self, container_name):
         """
         @summary: Get critical group and process lists by parsing the
diff --git a/tests/common/plugins/sanity_check/checks.py b/tests/common/plugins/sanity_check/checks.py
@@ -7,6 +7,7 @@
 
 logger = logging.getLogger(__name__)
 SYSTEM_STABILIZE_MAX_TIME = 300
+MONIT_STABILIZE_MAX_TIME = 420
 OMEM_THRESHOLD_BYTES=10485760 # 10MB
 
 def check_services(dut):
@@ -171,6 +172,80 @@ def check_dbmemory(dut):
     logger.info("Done checking database memory")
     return check_result
 
+def check_monit_services_status(check_result, monit_services_status):
+    """
+    @summary: Check whether each type of service which was monitored by Monit was in correct status or not.
+              If a service was in "Not monitored" status, sanity check will skip it since this service
+              was temporarily set to not be monitored by Monit.
+    @return: A dictionary contains the testing result (failed or not failed) and the status of each service.
+    """
+    check_result["services_status"] = {}
+    for service_name, service_info in monit_services_status.items():
+        check_result["services_status"].update({service_name: service_info["service_status"]})
+        if service_info["service_status"] == "Not monitored":
+            continue
+        if ((service_info["service_type"] == "Filesystem" and service_info["service_status"] != "Accessible")
+            or (service_info["service_type"] == "Process" and service_info["service_status"] != "Running")
+            or (service_info["service_type"] == "Program" and service_info["service_status"] != "Status ok")):
+            check_result["failed"] = True
+
+    return check_result
+
+def check_monit(dut):
+    """
+    @summary: Check whether the Monit is running and whether the services which were monitored by Monit are 
+              in the correct status or not.
+    @return: A dictionary contains the testing result (failed or not failed) and the status of each service.
+    """
+    logger.info("Checking status of each Monit service...")
+    networking_uptime = dut.get_networking_uptime().seconds
+    timeout = max((MONIT_STABILIZE_MAX_TIME - networking_uptime), 0)
+    interval = 20
+    logger.info("networking_uptime = {} seconds, timeout = {} seconds, interval = {} seconds" \
+                .format(networking_uptime, timeout, interval))
+
+    check_result = {"failed": False, "check_item": "monit"}
+
+    if timeout == 0:
+        monit_services_status = dut.get_monit_services_status()
+        if not monit_services_status:
+            logger.info("Monit was not running.")
+            check_result["failed"] = True
+            check_result["failed_reason"] = "Monit was not running"
+            logger.info("Checking status of each Monit service was done!")
+            return check_result
+
+        check_result = check_monit_services_status(check_result, monit_services_status)
+    else:
+        start = time.time()
+        elapsed = 0
+        is_monit_running = False
+        while elapsed < timeout:
+            check_result["failed"] = False
+            monit_services_status = dut.get_monit_services_status()
+            if not monit_services_status:
+                wait(interval, msg="Monit was not started and wait {} seconds to retry. Remaining time: {}." \
+                    .format(interval, timeout - elapsed))
+                elapsed = time.time() - start
+                continue
+
+            is_monit_running = True
+            check_result = check_monit_services_status(check_result, monit_services_status)
+            if check_result["failed"]:
+                wait(interval, msg="Services were not monitored and wait {} seconds to retry. Remaining time: {}. Services status: {}" \
+                    .format(interval, timeout - elapsed, str(check_result["services_status"])))
+                elapsed = time.time() - start
+            else:
+                break
+
+        if not is_monit_running:
+            logger.info("Monit was not running.")
+            check_result["failed"] = True
+            check_result["failed_reason"] = "Monit was not running"
+
+    logger.info("Checking status of each Monit service was done!")
+    return check_result
+
 def check_processes(dut):
     logger.info("Checking process status on %s..." % dut.hostname)
 
@@ -193,6 +268,7 @@ def check_processes(dut):
         start = time.time()
         elapsed = 0
         while elapsed < timeout:
+            check_result["failed"] = False
             processes_status = dut.all_critical_process_status()
             check_result["processes_status"] = processes_status
             check_result["services_status"] = {}
@@ -228,6 +304,8 @@ def do_checks(duthosts, check_items):
             elif item == "bgp":
                 if dut in duthosts.frontend_nodes:
                     results[dut.hostname].append(check_bgp_status(dut))
+            elif item == "monit":
+                results[dut.hostname].append(check_monit(dut))
 
     return results
 
diff --git a/tests/common/plugins/sanity_check/constants.py b/tests/common/plugins/sanity_check/constants.py
@@ -21,5 +21,5 @@
     "adaptive": {"cmd": None, "reboot": False, "adaptive": True, 'recover_wait': 30},
 }       # All supported recover methods
 
-SUPPORTED_CHECK_ITEMS = ["services", "interfaces", "dbmemory", "processes", "bgp"]          # Supported checks
-DEFAULT_CHECK_ITEMS = ["services", "interfaces", "dbmemory", "processes", "bgp"]            # Default checks
+SUPPORTED_CHECK_ITEMS = ["services", "interfaces", "dbmemory", "processes", "bgp", "monit"]          # Supported checks
+DEFAULT_CHECK_ITEMS = ["services", "interfaces", "dbmemory", "processes", "bgp", "monit"]            # Default checks
