1*** Settings ***
2Documentation  Verify Auto Restart policy for set of mission critical
3...            services needed for functioning on BMC.
4
5Resource         ../../lib/resource.robot
6Resource         ../../lib/connection_client.robot
7Resource         ../../lib/openbmc_ffdc.robot
8Resource         ../../lib/utils.robot
9Library          ../../data/platform_variables.py
10
11Suite Setup      Open Connection And Log In
12Suite Teardown   Close All Connections
13Test Teardown    FFDC On Test Case Fail
14
15*** Variables ***
16${LOG_SERVICE}  xyz.openbmc_project.Logging.service
17
18*** Test Cases ***
19
20Verify OpenBMC Services Auto Restart Policy
21    [Documentation]  Kill active services and expect auto restart.
22    [Tags]  Verify_OpenBMC_Services_Auto_Restart_Policy
23    # The services listed below restart policy should be "always"
24    # Command output:
25    # systemctl -p Restart show xyz.openbmc_project.Logging.service | cat
26    # Restart=always
27    @{services}=
28    ...  Create List  xyz.openbmc_project.Logging.service
29    ...               xyz.openbmc_project.ObjectMapper.service
30    ...               xyz.openbmc_project.State.BMC.service
31    ...               xyz.openbmc_project.State.Chassis.service
32    ...               xyz.openbmc_project.State.Host.service
33    FOR  ${SERVICE}  IN  @{services}
34      Check Service Autorestart  ${SERVICE}
35    END
36
37
38Kill Services And Expect Service Restart
39    [Documentation]  Kill the service and it must restart.
40    [Tags]  Kill_Services_And_Expect_Service_Restart
41
42    # Get the MainPID and service state.
43    ${MainPID}=  Get Service Attribute  MainPID  ${LOG_SERVICE}
44    Should Not Be Equal  ${0}  ${MainPID}
45    ...  msg=Logging service not restarted.
46
47    ${ActiveState}=  Get Service Attribute  ActiveState  ${LOG_SERVICE}
48    Should Be Equal  active  ${ActiveState}
49    ...  msg=Logging Service not in active state.
50
51    BMC Execute Command  kill -9 ${MainPID}
52    Sleep  10s  reason=Wait for service to restart.
53
54    ${MainPID}=  Get Service Attribute  MainPID  ${LOG_SERVICE}
55    Should Not Be Equal  ${0}  ${MainPID}
56    ...  msg=Logging service not restarted.
57
58    ${ActiveState}=  Get Service Attribute  ActiveState  ${LOG_SERVICE}
59    Should Be Equal  active  ${ActiveState}
60    ...  msg=Logging service not in active state.
61
62Kill The List Of Services And Expect Killed Service Gets Restarted
63    [Documentation]  Kill the given services and expect again services get restarted automatically.
64    [Tags]  Kill_The_List_Of_Services_And_Expect_Killed_Service_Gets_Restarted
65
66    # Create a list of services in respective server model python file
67    # like romulus.py, witherspoon.py on openbmc-test-automation/data directory etc.
68    # Example of creating a list of services in their respective server model python file
69    # SERVICES = {
70    # "BMC_SERVICES": ['xyz.openbmc_project.Logging.service', 'xyz.openbmc_project.ObjectMapper.service',
71    # 'xyz.openbmc_project.State.BMC.service', 'xyz.openbmc_project.State.Chassis.service',
72    # 'xyz.openbmc_project.State.Host.service']
73
74    @{auto_restart_policy_always_services}=  Create List
75    @{incorrect_auto_restart_policy_services}=  Create List
76    @{service_not_started}=  Create List
77
78    # Creating an list of services which needs to be validated.
79
80    ${services}=  Get Service Restart Policy Services  ${OPENBMC_MODEL}
81    ${service_list}=  Get From Dictionary  ${services}  BMC_SERVICES
82    ${length_services}=  Get Length  ${service_list}
83
84    # From service list it will check service auto-restart policy
85    # If incorrect those services will be appended to incorrect_auto_restart_policy_services list
86    # Proper restart policy services will be appended to auto_restart_policy_always_services list.
87
88    FOR  ${service}  IN  @{service_list}
89      ${service_status}=  Run Keyword And Return Status  Check Service Autorestart  ${service}
90      Run Keyword If  ${service_status} == False
91      ...    Append To List  ${incorrect_auto_restart_policy_services}  ${service}
92      ...  ELSE
93      ...    Append To List  ${auto_restart_policy_always_services}  ${service}
94    END
95
96    ${length_incorrect_autorestart_policy}=  Get Length  ${incorrect_auto_restart_policy_services}
97    Run Keyword If  ${length_incorrect_autorestart_policy} != 0 and ${length_incorrect_autorestart_policy} == ${length_services}
98    ...    Run Keywords  Log  ${incorrect_auto_restart_policy_services}  AND
99    ...    Fail  msg=All the given services have incorrect auto-restart policy.
100    ...  ELSE IF  ${length_incorrect_autorestart_policy} != 0 and ${length_incorrect_autorestart_policy} != ${length_services}
101    ...    Run Keywords  Log  ${incorrect_auto_restart_policy_services}  AND
102    ...    Run Keyword And Continue On Failure  Fail  msg=Listed services are having incorrect auto-restart policy.
103
104    # This will get process id and check the service active state before killing the services.
105    # If service process id was 0 or service was not in active state then those services will get
106    # appended to service_not_started list.
107    # Only services with process ID and in active state get killed and checked whether
108    # they automatically restart and put into active state.
109
110    FOR  ${service}  IN  @{auto_restart_policy_always_services}
111      ${Old_MainPID}=  Get Service Attribute  MainPID  ${service}
112      ${ActiveState}=  Get Service Attribute  ActiveState  ${service}
113      ${main_pid_status}=  Run Keyword And Return Status  Should Not Be Equal  ${0}  ${Old_MainPID}
114      ${active_state_status}=  Run Keyword And Return Status  Should Be Equal  active  ${ActiveState}
115      Run Keyword If  ${main_pid_status} == False or ${active_state_status} == False
116      ...  Run Keywords  Append To List  ${service_not_started}  ${service}  AND  Continue For Loop
117
118      BMC Execute Command  kill -9 ${Old_MainPID}
119      Sleep  10s  reason=Wait for service to restart.
120
121      ${New_MainPID}=  Get Service Attribute  MainPID  ${service}
122      Run Keyword And Continue On Failure  Should Not Be Equal  ${0}  ${New_MainPID}
123      ...  msg=${service} service not restarted.
124      Run Keyword And Continue On Failure  Should Not Be Equal  ${Old_MainPID}  ${New_MainPID}
125      ...  msg=Old process ID is mapped to ${service} service after service restart..
126
127      ${ActiveState}=  Get Service Attribute  ActiveState  ${service}
128      Run Keyword And Continue On Failure  Should Be Equal  active  ${ActiveState}
129      ...  msg=${service} service not in active state.
130    END
131
132    ${length_service_not_started}=  Get Length  ${service_not_started}
133    ${incorrect_services}=  Evaluate  ${length_incorrect_autorestart_policy} + ${length_service_not_started}
134
135    Run Keyword If  ${incorrect_services} == ${length_services} and ${length_service_not_started} != 0
136    ...    Run Keywords  Log  ${service_not_started}  AND  Fail  msg=All the services were either not started or not in active state by default.
137    ...  ELSE IF  ${incorrect_services} != ${length_services} and ${length_service_not_started} != 0
138    ...    Run Keywords  Log  ${service_not_started}  AND  Fail  msg=Few listed services were either not started or not in active state by default.
139
140*** Keywords ***
141
142Check Service Autorestart
143    [Documentation]  Check if given policy is "always".
144    [Arguments]  ${servicename}
145    # servicename  Qualified service name
146    ${restart_policy}=  Get Service Attribute  Restart  ${servicename}
147    Should Be Equal  always  ${restart_policy}
148    ...  msg=Incorrect policy for ${servicename}
149