1*** Settings ***
2
3Documentation       Test dump functionality of OpenBMC.
4
5Resource            ../../lib/openbmc_ffdc.robot
6Resource            ../../lib/rest_client.robot
7Resource            ../../lib/dump_utils.robot
8Resource            ../../lib/boot_utils.robot
9Resource            ../../lib/utils.robot
10Resource            ../../lib/state_manager.robot
11Library             ../../lib/bmc_ssh_utils.py
12
13Test Setup          Open Connection And Log In
14Test Teardown       Test Teardown Execution
15
16*** Test Cases ***
17
18Pre Dump BMC Performance Test
19    [Documentation]  Check performance of memory, CPU & file system of BMC.
20    [Tags]  Pre_Dump_BMC_Performance_Test
21
22    Open Connection And Log In
23    Check BMC Performance
24
25
26Verify User Initiated BMC Dump When Powered Off
27    [Documentation]  Create user initiated BMC dump at host off state and
28    ...  verify dump entry for it.
29    [Tags]  Verify_User_Initiated_BMC_Dump_When_Powered_Off
30
31    Redfish Power Off  stack_mode=skip
32    ${dump_id}=  Create User Initiated Dump
33    Check Existence Of BMC Dump File  ${dump_id}
34
35
36Verify Dump Persistency On Service Restart
37    [Documentation]  Create user dump, restart BMC service and verify dump
38    ...  persistency.
39    [Tags]  Verify_Dump_Persistency_On_Service_Restart
40
41    Delete All BMC Dump
42    ${dump_id}=  Create User Initiated Dump
43    BMC Execute Command
44    ...  systemctl restart xyz.openbmc_project.Dump.Manager.service
45    Sleep  10s  reason=Wait for BMC dump service to restart properly.
46
47    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}list
48    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
49    Check Existence Of BMC Dump File  ${dump_id}
50
51
52Verify Dump Persistency On Reset
53    [Documentation]  Create user dump, reset BMC and verify dump persistency.
54    [Tags]  Verify_Dump_Persistency_On_Reset
55
56    Delete All BMC Dump
57    ${dump_id}=  Create User Initiated Dump
58    OBMC Reboot (off)
59    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}list
60    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
61    Check Existence Of BMC Dump File  ${dump_id}
62
63
64Delete User Initiated BMC Dump And Verify
65    [Documentation]  Delete user initiated dump and verify.
66    [Tags]  Delete_User_Initiated_Dump_And_Verify
67
68    ${dump_id}=  Create User Initiated Dump
69    Check Existence Of BMC Dump File  ${dump_id}
70
71    Delete BMC Dump  ${dump_id}
72
73
74Verify User Initiated Dump Size
75    [Documentation]  Verify user Initiated BMC dump size is under 200k.
76    [Tags]  Verify_User_Initiated_Dump_Size
77
78    ${dump_id}=  Create User Initiated Dump
79
80    ${dump_size}=  Read Attribute  ${DUMP_ENTRY_URI}${dump_id}  Size
81    # Max size for dump is 200k = 200x1024
82    Should Be True  0 < ${dump_size} < 204800
83    Check Existence Of BMC Dump File  ${dump_id}
84
85
86Create Two User Initiated Dump And Delete One
87    [Documentation]  Create two dumps and delete the first.
88    [Tags]  Create_Two_User_Initiated_Dump_And_Delete_One
89
90    ${dump_id_1}=  Create User Initiated Dump
91    ${dump_id_2}=  Create User Initiated Dump
92
93    Delete BMC Dump  ${dump_id_1}
94
95    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}${dump_id_1}
96    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
97
98    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}${dump_id_2}
99    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
100    Check Existence Of BMC Dump File  ${dump_id_2}
101
102
103Create And Delete BMC Dump Multiple Times
104    [Documentation]  Create and delete BMC dump multiple times.
105    [Tags]  Create_And_Delete_BMC_Dump_Multiple_Times
106
107    FOR  ${INDEX}  IN  1  5
108      ${dump_id}=  Create User Initiated Dump
109      Delete BMC Dump  ${dump_id}
110    END
111
112
113Delete All BMC Dumps And Verify
114    [Documentation]  Delete all BMC dumps and verify.
115    [Tags]  Delete_All_BMC_Dumps_And_Verify
116
117    # Create some dump.
118    Create User Initiated Dump
119    Create User Initiated Dump
120
121    Delete All BMC Dump
122    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}list
123    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
124
125
126Verify User Initiated BMC Dump When Host Booted
127    [Documentation]  Create user initiated BMC dump at host booted state and
128    ...  verify dump entry for it.
129    [Tags]  Verify_User_Initiated_BMC_Dump_When_Host_Booted
130
131    Redfish Power On  stack_mode=skip
132    Create User Initiated Dump
133
134
135Verify Core Dump Size
136    [Documentation]  Verify BMC core dump size is under 200k.
137    [Tags]  Verify_Core_Dump_Size
138
139    Delete All Dumps
140    Trigger Core Dump
141    Wait Until Keyword Succeeds  2 min  10 sec  Get Dump Entries
142
143    ${dump_entries}=  Get URL List  ${DUMP_ENTRY_URI}
144    ${dump_size}=  Read Attribute  ${dump_entries[0]}  Size
145
146    # Max size for dump is 200k = 200x1024
147    Should Be True  0 < ${dump_size} < 204800  msg=Size of dump is incorrect.
148
149
150Dump Out Of Space Test
151    [Documentation]  Verify out of dump space is reported when attempt
152    ...  to create too many dumps.
153    [Tags]  Dump_Out_Of_Space_Test
154    [Setup]  Delete All BMC Dump
155
156    # Systems typically hold 8-14 dumps before running out of dump space.
157    # Attempt to create too_many_dumps.  Expect to run out of space
158    # before this.
159    ${too_many_dumps}  Set Variable  ${20}
160
161    # Loop, creating a dump each iteration.  Will either get dump_id or
162    # will get EMPTY when out of dump space.
163    FOR  ${n}  IN RANGE  0  ${too_many_dumps}
164      ${dump_id}=  Create User Initiated Dump  check_out_of_space=${True}
165      Exit For Loop If  '${dump_id}' == '${EMPTY}'
166      Check Existence Of BMC Dump File  ${dump_id}
167    END
168
169    Run Keyword If  '${dump_id}' != '${EMPTY}'  Fail
170    ...  msg=Did not run out of dump space as expected.
171
172
173Post Dump BMC Performance Test
174    [Documentation]  Check performance of memory, CPU & file system of BMC.
175    [Tags]  Post_Dump_BMC_Performance_Test
176
177    Open Connection And Log In
178    Check BMC Performance
179
180
181Post Dump Core Dump Check
182    [Documentation]  Check core dump existence on BMC after code update.
183    [Tags]  Post_Dump_Core_Dump_Check
184
185    Check For Core Dumps
186
187
188Verify Dump After Host Watchdog Error Injection
189    [Documentation]  Inject host watchdog error and verify whether dump is generated.
190    [Tags]  Verify_Dump_After_Host_Watchdog_Error_Injection
191
192    Redfish Power On
193
194    Run Keyword And Ignore Error  Delete All Dumps
195
196    # Enable auto reboot
197    Set Auto Reboot  ${1}
198
199    Trigger Host Watchdog Error  2000  30
200
201    Wait Until Keyword Succeeds  300 sec  20 sec  Is Host Rebooted
202
203    #Get dump details
204    @{dump_entry_list}=  Read Properties  ${DUMP_ENTRY_URI}
205
206    # Verifing that there is only one dump
207    ${length}=  Get length  ${dump_entry_list}
208    Should Be Equal As Integers  ${length}  ${1}
209
210    # Get dump id
211    ${value}=  Get From List  ${dump_entry_list}  0
212    @{split_value}=  Split String  ${value}  /
213    ${dump_id}=  Get From List  ${split_value}  -1
214
215    # Max size for dump is 200k = 200x1024
216    ${dump_size}=  Read Attribute  ${DUMP_ENTRY_URI}${dump_id}  Size
217    Should Be True  0 < ${dump_size} < 204800
218
219
220Verify Download BMC Dump
221    [Documentation]  Verify that a BMC dump can be downloaded to the local machine.
222    [Tags]  Verify_Download_BMC_Dump
223
224    ${dump_id}=  Create User Initiated Dump
225    ${dump_dict}=  Get Dump Dict
226    ${bmc_dump_name}=  Fetch From Right  ${dump_dict['${dump_id}']}  /
227    ${bmc_dump_checksum}  ${stderr}  ${rc}=  BMC Execute Command
228    ...  md5sum ${dump_dict['${dump_id}']}|awk '{print$1}'
229    ${bmc_dump_size}  ${stderr}  ${rc}=  BMC Execute Command
230    ...  stat -c "%s" ${dump_dict['${dump_id}']}
231
232    ${response}=  OpenBMC Get Request  ${DUMP_DOWNLOAD_URI}${dump_id}
233    ...  quiet=${1}
234    Should Be Equal As Strings  ${response.status_code}  ${HTTP_OK}
235    Create Binary File  ${EXECDIR}${/}dumps   ${response.content}
236    Run  tar -xvf ${EXECDIR}${/}dumps
237    ${download_dump_name}=  Fetch From Left  ${bmc_dump_name}  .
238    ${download_dump_checksum}=  Run  md5sum ${EXECDIR}/dumps|awk '{print$1}'
239    ${download_dump_size}=  Run  stat -c "%s" ${EXECDIR}${/}dumps
240
241    OperatingSystem.Directory Should Exist  ${EXECDIR}/${download_dump_name}
242    ...  msg=Created dump name and downloaded dump name don't match.
243    Should Be Equal As Strings  ${bmc_dump_checksum}  ${download_dump_checksum}
244    Should Be Equal As Strings  ${bmc_dump_size}  ${download_dump_size}
245
246    Run  rm -rf ${EXECDIR}${/}${download_dump_name};rm ${EXECDIR}${/}dumps
247
248
249*** Keywords ***
250
251Test Teardown Execution
252    [Documentation]  Do the post test teardown.
253
254    Wait Until Keyword Succeeds  3 min  15 sec  Verify No Dump In Progress
255    FFDC On Test Case Fail
256    Delete All BMC Dump
257    Close All Connections
258