1*** Settings ***
2Documentation  This module provides general keywords for dump.
3
4Library         bmc_ssh_utils.py
5Variables       ../data/variables.py
6
7*** Variables ***
8
9*** Keywords ***
10
11Create User Initiated Dump
12    [Documentation]  Generate user initiated dump and return
13    ...  the dump id number (e.g., "5").  Optionally return EMPTY
14    ...  if out of dump space.
15    [Arguments]   ${check_out_of_space}=${False}
16
17    # Description of Argument(s):
18    # check_out_of_space   If ${False}, a dump will be created and
19    #                      its dump_id will be returned.
20    #                      If ${True}, either the dump_id will be
21    #                      returned, or the value ${EMPTY} will be
22    #                      returned if out of dump space was
23    #                      detected when creating the dump.
24
25    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
26    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
27    ...  Set Global Variable  ${REST_DUMP_URI}  /xyz/openbmc_project/dump/
28
29    ${data}=  Create Dictionary  data=@{EMPTY}
30    ${resp}=  OpenBMC Post Request
31    ...  ${REST_DUMP_URI}action/CreateDump  data=${data}  quiet=${1}
32
33    Run Keyword If  '${check_out_of_space}' == '${False}'
34    ...      Run Keyword And Return  Get The Dump Id  ${resp}
35    ...  ELSE
36    ...      Run Keyword And Return  Check For Too Many Dumps  ${resp}
37
38
39Get The Dump Id
40    [Documentation]  Wait for the dump to be created. Return the
41    ...  dump id number (e.g., "5").
42    [Arguments]  ${resp}
43
44    # Description of Argument(s):
45    # resp   Response object from action/Create Dump attempt.
46    #        Example object:
47    #        {
48    #           "data": 5,
49    #           "message": "200 OK",
50    #           "status": "ok"
51    #        },
52    #        The "data" field conveys the id number of the created dump.
53
54    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
55
56    Run Keyword If  ${resp.json()["data"]} == ${None}
57    ...  Fail  Dump id returned null.
58
59    ${dump_id}=  Set Variable  ${json["data"]}
60
61    Wait Until Keyword Succeeds  3 min  15 sec  Check Dump Existence
62    ...  ${dump_id}
63
64    [Return]  ${dump_id}
65
66
67Check For Too Many Dumps
68    [Documentation]  Return the dump_id number, or return ${EMPTY} if dump
69    ...  creation failed due to too many dumps.
70    [Arguments]  ${resp}
71
72    # Description of Argument(s):
73    # resp   Response object from action/Create Dump attempt.
74    #        Example object if there are too many dumps:
75    #       {
76    #           "data": {
77    #               "description": "xyz.openbmc_project.Dump.Create.Error.QuotaExceeded"
78    #           },
79    #           "message": "Dump not captured due to a cap.",
80    #           "status": "error"
81    #       }
82
83    # If dump was created normally, return the dump_id number.
84    Run Keyword If  '${resp.status_code}' == '${HTTP_OK}'
85    ...  Run Keyword And Return  Get The Dump Id  ${resp}
86
87    ${exception}=  Set Variable  ${resp.json()["message"]}
88    ${at_capacity}=  Set Variable  Dump not captured due to a cap
89    ${too_many_dumps}=  Evaluate  $at_capacity in $exception
90    Printn
91    Rprint Vars   exception  too_many_dumps
92    # If there are too many dumps, return ${EMPTY}, otherwise Fail.
93    ${status}=  Run Keyword If  ${too_many_dumps}  Set Variable  ${EMPTY}
94    ...  ELSE  Fail  msg=${exception}.
95
96    [Return]  ${status}
97
98
99Verify No Dump In Progress
100    [Documentation]  Verify no dump in progress.
101
102    ${dump_progress}  ${stderr}  ${rc}=  BMC Execute Command  ls /tmp
103    Should Not Contain  ${dump_progress}  obmcdump
104
105
106Check Dump Existence
107    [Documentation]  Verify if given dump exist.
108    [Arguments]  ${dump_id}
109
110    # Description of Argument(s):
111    # dump_id  An integer value that identifies a particular dump
112    #          object(e.g. 1, 3, 5).
113
114    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
115    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
116    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
117
118    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}${dump_id}
119    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
120
121
122Delete BMC Dump
123    [Documentation]  Deletes a given bmc dump.
124    [Arguments]  ${dump_id}
125
126    # Description of Argument(s):
127    # dump_id  An integer value that identifies a particular dump (e.g. 1, 3).
128
129    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
130    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
131    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
132
133    ${args}=  Set Variable   {"data": []}
134    ${resp}=  OpenBMC Post Request
135    ...  ${DUMP_ENTRY_URI}${dump_id}/action/Delete  data=${args}
136
137    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
138
139Delete All Dumps
140    [Documentation]  Delete all dumps.
141
142    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
143    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
144    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
145
146    # Check if dump entries exist, if not return.
147    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}list  quiet=${1}
148    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
149
150    # Get the list of dump entries and delete them all.
151    ${dump_entries}=  Get URL List  ${DUMP_ENTRY_URI}
152    FOR  ${entry}  IN  @{dump_entries}
153        ${dump_id}=  Fetch From Right  ${entry}  /
154        Delete BMC Dump  ${dump_id}
155    END
156
157
158Redfish Delete BMC Dump
159    [Documentation]  Deletes a given BMC dump via Redfish.
160    [Arguments]  ${dump_id}
161
162    # Description of Argument(s):
163    # dump_id  An integer value that identifies a particular dump (e.g. 1, 3).
164
165    Redfish.Delete  /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Entries/${dump_id}
166
167
168Redfish Delete All BMC Dumps
169    [Documentation]  Delete all BMC dumps via Redfish.
170
171    # Check if dump entries exist, if not return.
172    ${resp}=  Redfish.Get  /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Entries
173    Return From Keyword If  ${resp.dict["Members@odata.count"]} == ${0}
174
175    Redfish.Post  /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Actions/LogService.ClearLog
176
177
178Redfish Get All System Dumps
179     [Documentation]  Get the system dump log entries.
180
181     ${resp}=  Redfish.Get  ${REDFISH_SYSTEM_DUMP}
182
183     [Return]  ${resp.dict}
184
185
186Get Redfish BMC Dump Log Entries
187     [Documentation]  Get the BMC dump log entries.
188
189     ${resp}=  Redfish.Get  ${REDFISH_DUMP_URI}
190
191     [Return]  ${resp.dict}
192
193
194Redfish Delete All System Dumps
195    [Documentation]  Delete all system  dumps via Redfish.
196
197    Redfish.Post  /redfish/v1/Systems/system/LogServices/Dump/Actions/LogService.ClearLog
198
199
200Redfish BMC Dump Should Not Exist
201     [Documentation]  Verify that there is no BMC dump at dump URI.
202
203     # Verify no dump exists.
204     ${dump_entries}=  Get Redfish BMC Dump Log Entries
205     Should Be Equal As Integers  0  ${dump_entries['Members@odata.count']}
206
207
208Redfish BMC Dump Should Exist
209    [Documentation]  Check if BMC dump is generated.
210
211    ${dump_entries}=  Get Redfish BMC Dump Log Entries
212    Should Be True   ${dump_entries['Members@odata.count']} >= 1  msg=No BMC dump generated.
213
214
215Delete All BMC Dump
216    [Documentation]  Delete all BMC dump entries using "DeleteAll" interface.
217
218    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
219    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
220    ...  Set Global Variable  ${REST_DUMP_URI}  /xyz/openbmc_project/dump/
221
222    ${args}=  Set Variable   {"data": []}
223    ${resp}=  Openbmc Post Request  ${REST_DUMP_URI}action/DeleteAll  data=${args}
224    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
225
226Dump Should Not Exist
227    [Documentation]  Verify that BMC dumps do not exist.
228
229    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
230    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
231    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
232
233    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}list  quiet=${1}
234    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
235
236Check Existence Of BMC Dump File
237    [Documentation]  Verify existence of BMC dump file.
238    [Arguments]  ${dump_id}
239
240    # Description of argument(s):
241    # dump_id  BMC dump identifier
242
243    ${dump_check_cmd}=  Set Variable
244    ...  ls /var/lib/phosphor-debug-collector/dumps
245
246    # Output of sample BMC Execute command with '2' as dump id is as follows
247    # ls /var/lib/phosphor-debug-collector/dumps/2
248    # obmcdump_2_XXXXXXXXXX.tar.xz
249    ${file_there}  ${stderr}  ${rc}=  BMC Execute Command
250    ...  ${dump_check_cmd}/${dump_id}
251    Should End With  ${file_there}  tar.xz  msg=BMC dump file not found.
252
253Get Dump Entries
254    [Documentation]  Return dump entries list.
255
256    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
257    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
258    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
259
260    ${dump_entries}=  Get URL List  ${DUMP_ENTRY_URI}
261    [Return]  ${dump_entries}
262
263Trigger Core Dump
264    [Documentation]  Trigger core dump.
265
266    # Find the pid of the active ipmid and kill it.
267    ${cmd_buf}=  Catenate  kill -s SEGV $(ps | egrep ' ipmid$' |
268    ...  egrep -v grep | \ cut -c1-6)
269
270    ${cmd_output}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd_buf}
271    Should Be Empty  ${stderr}  msg=BMC execute command error.
272    Should Be Equal As Integers  ${rc}  ${0}
273    ...  msg=BMC execute command return code is not zero.
274
275Initiate BMC Dump Using Redfish And Return Task Id
276     [Documentation]  Initiate BMC dump via Redfish and return its task ID.
277
278     ${payload}=  Create Dictionary  DiagnosticDataType=Manager
279     ${resp}=  Redfish.Post
280     ...  /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Actions/LogService.CollectDiagnosticData
281     ...  body=${payload}  valid_status_codes=[${HTTP_ACCEPTED}]
282
283     # Example of response from above Redfish POST request.
284     # "@odata.id": "/redfish/v1/TaskService/Tasks/0",
285     # "@odata.type": "#Task.v1_4_3.Task",
286     # "Id": "0",
287     # "TaskState": "Running",
288     # "TaskStatus": "OK"
289
290     [Return]  ${resp.dict['Id']}
291
292Create User Initiated BMC Dump Via Redfish
293    [Documentation]  Generate user initiated BMC dump via Redfish and return the dump id number (e.g., "5").
294    [Arguments]  ${skip_dump_completion}=0
295
296    # Description of Argument(s):
297    # skip_dump_completion          If skip_dump_completion is set to 0, this
298    #                               keyword will waiting for BMC dump to
299    #                               complete and returns the dump id.
300    #                               Otherwise, the keyword is skipped after
301    #                               initiating BMC dump and returns dump task id.
302
303    ${payload}=  Create Dictionary  DiagnosticDataType=Manager
304    ${resp}=  Redfish.Post  /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Actions/LogService.CollectDiagnosticData
305    ...  body=${payload}  valid_status_codes=[${HTTP_ACCEPTED}]
306
307    # Example of response from above Redfish POST request.
308    # "@odata.id": "/redfish/v1/TaskService/Tasks/0",
309    # "@odata.type": "#Task.v1_4_3.Task",
310    # "Id": "0",
311    # "TaskState": "Running",
312    # "TaskStatus": "OK"
313
314    Run Keyword If  ${skip_dump_completion} != 0  Return From Keyword  ${resp.dict['Id']}
315    Wait Until Keyword Succeeds  5 min  15 sec  Check Task Completion  ${resp.dict['Id']}
316    ${task_id}=  Set Variable  ${resp.dict['Id']}
317
318    ${task_dict}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
319
320    # Example of HttpHeaders field of task details.
321    # "Payload": {
322    #   "HttpHeaders": [
323    #     "Host: <BMC_IP>",
324    #      "Accept-Encoding: identity",
325    #      "Connection: Keep-Alive",
326    #      "Accept: */*",
327    #      "Content-Length: 33",
328    #      "Location: /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Entries/2"]
329    #    ],
330    #    "HttpOperation": "POST",
331    #    "JsonBody": "{\"DiagnosticDataType\":\"Manager\"}",
332    #     "TargetUri": "/redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Actions/LogService.CollectDiagnosticData"
333    # }
334
335    [Return]  ${task_dict["Payload"]["HttpHeaders"][-1].split("/")[-1]}
336
337Auto Generate BMC Dump
338    [Documentation]  Auto generate BMC dump.
339
340    ${cmd}=  Catenate  busctl --verbose call xyz.openbmc_project.Dump.Manager
341    ...  /xyz/openbmc_project/dump/bmc xyz.openbmc_project.Dump.Create CreateDump a{sv} 0
342    ${stdout}  ${stderr}  ${rc}=
343    ...  BMC Execute Command  ${cmd}
344    [Return]  ${stdout}  ${stderr}  ${rc}
345
346Get Dump Size
347    [Documentation]  Get dump size.
348    [Arguments]  ${dump_uri}
349
350    # Description of argument(s):
351    # dump_uri        Dump URI
352    #                 (Eg. 	/xyz/openbmc_project/dump/bmc/entry/1).
353
354    # Example of Dump entry.
355    # "data": {
356    #   "CompletedTime": 1616760931,
357    #   "Elapsed": 1616760931,
358    #   "OffloadUri": "",
359    #   "Offloaded": false,
360    #   "Password": "",
361    #   "Size": 3056,
362    #   "SourceDumpId": 117440513,
363    #   "StartTime": 1616760931,
364    #   "Status": "xyz.openbmc_project.Common.Progress.OperationStatus.Completed",
365    #   "VSPString": ""
366    #  },
367
368    Log  ${dump_uri}
369    ${dump_data}=  Redfish.Get Properties  ${dump_uri}
370    [Return]  ${dump_data["data"]["Size"]}
371
372Get Dump ID
373    [Documentation]  Return dump ID.
374    [Arguments]   ${task_id}
375
376    # Description of argument(s):
377    # task_id        Task ID.
378
379    # Example of HttpHeaders field of task details.
380    # "Payload": {
381    #   "HttpHeaders": [
382    #     "Host: <BMC_IP>",
383    #      "Accept-Encoding: identity",
384    #      "Connection: Keep-Alive",
385    #      "Accept: */*",
386    #      "Content-Length: 33",
387    #      "Location: /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Entries/2"]
388    #    ],
389    #    "HttpOperation": "POST",
390    #    "JsonBody": "{\"DiagnosticDataType\":\"Manager\"}",
391    #     "TargetUri":
392    # "/redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Actions/LogService.CollectDiagnosticData"
393    # }
394
395    ${task_dict}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
396    ${key}  ${value}=  Set Variable  ${task_dict["Payload"]["HttpHeaders"][-1].split(":")}
397    Run Keyword If  '${key}' != 'Location'  Fail
398    [Return]  ${value.strip('/').split('/')[-1]}
399
400Get Task Status
401    [Documentation]  Return task status.
402    [Arguments]   ${task_id}
403
404    # Description of argument(s):
405    # task_id        Task ID.
406
407    ${resp}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
408    [Return]  ${resp['TaskState']}
409
410Check Task Completion
411    [Documentation]  Check if the task is complete.
412    [Arguments]   ${task_id}
413
414    # Description of argument(s):
415    # task_id        Task ID.
416
417    ${task_dict}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
418    Should Be Equal As Strings  ${task_dict['TaskState']}  Completed
419
420Get Dump ID And Status
421    [Documentation]  Return dump ID and status.
422    [Arguments]   ${task_id}
423
424    # Description of argument(s):
425    # task_id        Task ID.
426
427    Wait Until Keyword Succeeds  10 min  15 sec  Check Task Completion  ${task_id}
428    ${dump_id}=  Get Dump ID  ${task_id}
429    [Return]  ${dump_id}  Completed
430
431
432Create BMC User Dump
433    [Documentation]  Generate user initiated BMC dump via Redfish and return
434    ...  the task instance Id and response object (e.g., "5").
435
436    ${payload}=  Create Dictionary  DiagnosticDataType=Manager
437    ${resp}=  Redfish.Post  /redfish/v1/Managers/${MANAGER_ID}/LogServices/Dump/Actions/LogService.CollectDiagnosticData
438    ...  body=${payload}  valid_status_codes=[${HTTP_ACCEPTED}]
439
440    ${ip_resp}=  Evaluate  json.loads(r'''${resp.text}''')  json
441
442    Return From Keyword  ${ip_resp["Id"]}  ${resp}
443
444
445Wait For Task Completion
446    [Documentation]  Check whether the state of task instance matches any of the
447    ...  expected completion states before maximum number of retries exceeds and
448    ...  exit loop in case completion state is met.
449    [Arguments]  ${task_id}  ${expected_status}  ${retry_max_count}=300
450    ...  ${check_state}=${FALSE}
451
452    # Description of argument(s):
453    # task_id                     the task id for which completion is
454    #                             to be monitored.
455    # expected_status             the task state which is to be considered as the
456    #                             end of task life cycle.
457    # retry_max_count             the maximum number of retry count to wait for
458    #                             task to reach its completion state. Default
459    #                             value of retry_max_count is 300.
460    # check_state                 if set as TRUE, the task state will be
461    #                             monitored whether the task state value is
462    #                             valid throughout task life cycle until
463    #                             expected completion state is reached.
464    #                             Default value of check_state is FALSE.
465
466    FOR  ${retry}  IN RANGE  ${retry_max_count}
467        ${resp}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
468        ${current_task_state}=  Set Variable  ${resp["TaskState"]}
469        Rprint Vars  current_task_state
470
471        Run Keyword If  ${check_state} == ${TRUE}  Should Be True
472        ...  '${resp["TaskState"]}' in ${allowed_task_state}
473        ...  msg=Verify task state is valid
474
475        Exit For Loop If
476        ...  '${resp["TaskState"]}' in ${expected_status}
477
478        Sleep  5s
479    END
480
481Get Dump Status In BMC
482    [Documentation]  Get dump status from BMC using busctl method.
483    [Arguments]  ${dump_uri}
484
485    # Description of argument(s):
486    # dump_uri   Dump URI E.g: /xyz/openbmc_project/dump/bmc/entry/7.
487
488    ${cmd}=  Catenate  busctl get-property xyz.openbmc_project.Dump.Manager
489    ...  ${dump_uri} xyz.openbmc_project.Common.Progress Status
490
491    ${stdout}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd}
492    Log  ${stdout}
493    # Example output:
494    # s "xyz.openbmc_project.Common.Progress.OperationStatus.Completed".
495
496    ${status}=  Set Variable  ${stdout.split('.')[-1].strip('"')}
497    [Return]  ${status}
498
499Verify Dump Status In BMC
500    [Documentation]  Verify Dump Status in BMC.
501    [Arguments]  ${dump_uri}  ${expected_dump_status}
502
503    # Description of argument(s):
504    # dump_uri              Dump URI E.g: /xyz/openbmc_project/dump/bmc/entry/7.
505    # expected_dump_status  Expected Dump Status (Completed or Failed etc).
506
507    ${dump_status}=  Get Dump Status In BMC  ${dump_uri}
508    Should Be Equal  ${dump_status}  ${expected_dump_status}
509