1*** Settings ***
2Documentation  This module provides general keywords for dump.
3
4Library         bmc_ssh_utils.py
5Variables       ../data/variables.py
6
7*** Variables ***
8
9*** Keywords ***
10
11Create User Initiated Dump
12    [Documentation]  Generate user initiated dump and return
13    ...  the dump id number (e.g., "5").  Optionally return EMPTY
14    ...  if out of dump space.
15    [Arguments]   ${check_out_of_space}=${False}
16
17    # Description of Argument(s):
18    # check_out_of_space   If ${False}, a dump will be created and
19    #                      its dump_id will be returned.
20    #                      If ${True}, either the dump_id will be
21    #                      returned, or the value ${EMPTY} will be
22    #                      returned if out of dump space was
23    #                      detected when creating the dump.
24
25    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
26    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
27    ...  Set Global Variable  ${REST_DUMP_URI}  /xyz/openbmc_project/dump/
28
29    ${data}=  Create Dictionary  data=@{EMPTY}
30    ${resp}=  OpenBMC Post Request
31    ...  ${REST_DUMP_URI}action/CreateDump  data=${data}  quiet=${1}
32
33    Run Keyword If  '${check_out_of_space}' == '${False}'
34    ...      Run Keyword And Return  Get The Dump Id  ${resp}
35    ...  ELSE
36    ...      Run Keyword And Return  Check For Too Many Dumps  ${resp}
37
38
39Get The Dump Id
40    [Documentation]  Wait for the dump to be created. Return the
41    ...  dump id number (e.g., "5").
42    [Arguments]  ${resp}
43
44    # Description of Argument(s):
45    # resp   Response object from action/Create Dump attempt.
46    #        Example object:
47    #        {
48    #           "data": 5,
49    #           "message": "200 OK",
50    #           "status": "ok"
51    #        },
52    #        The "data" field conveys the id number of the created dump.
53
54    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
55
56    Run Keyword If  ${resp.json()["data"]} == ${None}
57    ...  Fail  Dump id returned null.
58
59    ${dump_id}=  Set Variable  ${json["data"]}
60
61    Wait Until Keyword Succeeds  3 min  15 sec  Check Dump Existence
62    ...  ${dump_id}
63
64    [Return]  ${dump_id}
65
66
67Check For Too Many Dumps
68    [Documentation]  Return the dump_id number, or return ${EMPTY} if dump
69    ...  creation failed due to too many dumps.
70    [Arguments]  ${resp}
71
72    # Description of Argument(s):
73    # resp   Response object from action/Create Dump attempt.
74    #        Example object if there are too many dumps:
75    #       {
76    #           "data": {
77    #               "description": "xyz.openbmc_project.Dump.Create.Error.QuotaExceeded"
78    #           },
79    #           "message": "Dump not captured due to a cap.",
80    #           "status": "error"
81    #       }
82
83    # If dump was created normally, return the dump_id number.
84    Run Keyword If  '${resp.status_code}' == '${HTTP_OK}'
85    ...  Run Keyword And Return  Get The Dump Id  ${resp}
86
87    ${exception}=  Set Variable  ${resp.json()["message"]}
88    ${at_capacity}=  Set Variable  Dump not captured due to a cap
89    ${too_many_dumps}=  Evaluate  $at_capacity in $exception
90    Printn
91    Rprint Vars   exception  too_many_dumps
92    # If there are too many dumps, return ${EMPTY}, otherwise Fail.
93    ${status}=  Run Keyword If  ${too_many_dumps}  Set Variable  ${EMPTY}
94    ...  ELSE  Fail  msg=${exception}.
95
96    [Return]  ${status}
97
98
99Verify No Dump In Progress
100    [Documentation]  Verify no dump in progress.
101
102    ${dump_progress}  ${stderr}  ${rc}=  BMC Execute Command  ls /tmp
103    Should Not Contain  ${dump_progress}  obmcdump
104
105
106Check Dump Existence
107    [Documentation]  Verify if given dump exist.
108    [Arguments]  ${dump_id}
109
110    # Description of Argument(s):
111    # dump_id  An integer value that identifies a particular dump
112    #          object(e.g. 1, 3, 5).
113
114    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
115    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
116    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
117
118    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}${dump_id}
119    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
120
121
122Delete BMC Dump
123    [Documentation]  Deletes a given bmc dump.
124    [Arguments]  ${dump_id}
125
126    # Description of Argument(s):
127    # dump_id  An integer value that identifies a particular dump (e.g. 1, 3).
128
129    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
130    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
131    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
132
133    ${args}=  Set Variable   {"data": []}
134    ${resp}=  OpenBMC Post Request
135    ...  ${DUMP_ENTRY_URI}${dump_id}/action/Delete  data=${args}
136
137    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
138
139Delete All Dumps
140    [Documentation]  Delete all dumps.
141
142    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
143    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
144    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
145
146    # Check if dump entries exist, if not return.
147    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}list  quiet=${1}
148    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
149
150    # Get the list of dump entries and delete them all.
151    ${dump_entries}=  Get URL List  ${DUMP_ENTRY_URI}
152    FOR  ${entry}  IN  @{dump_entries}
153        ${dump_id}=  Fetch From Right  ${entry}  /
154        Delete BMC Dump  ${dump_id}
155    END
156
157
158Redfish Delete BMC Dump
159    [Documentation]  Deletes a given BMC dump via Redfish..
160    [Arguments]  ${dump_id}
161
162    # Description of Argument(s):
163    # dump_id  An integer value that identifies a particular dump (e.g. 1, 3).
164
165    Redfish.Delete  /redfish/v1/Managers/bmc/LogServices/Dump/Entries/${dump_id}
166
167
168Redfish Delete All BMC Dumps
169    [Documentation]  Delete all BMC dumps via Redfish.
170
171    # Check if dump entries exist, if not return.
172    ${resp}=  Redfish.Get  /redfish/v1/Managers/bmc/LogServices/Dump/Entries
173    Return From Keyword If  ${resp.dict["Members@odata.count"]} == ${0}
174
175    Redfish.Post  /redfish/v1/Managers/bmc/LogServices/Dump/Actions/LogService.ClearLog
176
177
178Get Redfish BMC Dump Log Entries
179     [Documentation]  Get the BMC dump log entries.
180
181     ${resp}=  Redfish.Get  ${REDFISH_DUMP_URI}
182
183     [Return]  ${resp.dict}
184
185
186Redfish Delete All System Dumps
187    [Documentation]  Delete all system  dumps via Redfish.
188
189    Redfish.Post  /redfish/v1/Systems/system/LogServices/Dump/Actions/LogService.ClearLog
190
191
192Redfish BMC Dump Should Not Exist
193     [Documentation]  Verify that there is no BMC dump at dump URI.
194
195     # Verify no dump exists.
196     ${dump_entries}=  Get Redfish BMC Dump Log Entries
197     Should Be Equal As Integers  0  ${dump_entries['Members@odata.count']}
198
199
200Delete All BMC Dump
201    [Documentation]  Delete all BMC dump entries using "DeleteAll" interface.
202
203    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
204    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
205    ...  Set Global Variable  ${REST_DUMP_URI}  /xyz/openbmc_project/dump/
206
207    ${args}=  Set Variable   {"data": []}
208    ${resp}=  Openbmc Post Request  ${REST_DUMP_URI}action/DeleteAll  data=${args}
209    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
210
211Dump Should Not Exist
212    [Documentation]  Verify that BMC dumps do not exist.
213
214    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
215    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
216    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
217
218    ${resp}=  OpenBMC Get Request  ${DUMP_ENTRY_URI}list  quiet=${1}
219    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
220
221Check Existence Of BMC Dump File
222    [Documentation]  Verify existence of BMC dump file.
223    [Arguments]  ${dump_id}
224
225    # Description of argument(s):
226    # dump_id  BMC dump identifier
227
228    ${dump_check_cmd}=  Set Variable
229    ...  ls /var/lib/phosphor-debug-collector/dumps
230
231    # Output of sample BMC Execute command with '2' as dump id is as follows
232    # ls /var/lib/phosphor-debug-collector/dumps/2
233    # obmcdump_2_XXXXXXXXXX.tar.xz
234    ${file_there}  ${stderr}  ${rc}=  BMC Execute Command
235    ...  ${dump_check_cmd}/${dump_id}
236    Should End With  ${file_there}  tar.xz  msg=BMC dump file not found.
237
238Get Dump Entries
239    [Documentation]  Return dump entries list.
240
241    ${resp}=  OpenBMC Get Request  ${REST_DUMP_URI}
242    Run Keyword If  '${resp.status_code}' == '${HTTP_NOT_FOUND}'
243    ...  Set Global Variable  ${DUMP_ENTRY_URI}  /xyz/openbmc_project/dump/entry/
244
245    ${dump_entries}=  Get URL List  ${DUMP_ENTRY_URI}
246    [Return]  ${dump_entries}
247
248Trigger Core Dump
249    [Documentation]  Trigger core dump.
250
251    # Find the pid of the active ipmid and kill it.
252    ${cmd_buf}=  Catenate  kill -s SEGV $(ps | egrep ' ipmid$' |
253    ...  egrep -v grep | \ cut -c1-6)
254
255    ${cmd_output}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd_buf}
256    Should Be Empty  ${stderr}  msg=BMC execute command error.
257    Should Be Equal As Integers  ${rc}  ${0}
258    ...  msg=BMC execute command return code is not zero.
259
260Initiate BMC Dump Using Redfish And Return Task Id
261     [Documentation]  Initiate BMC dump via Redfish and return its task ID.
262
263     ${payload}=  Create Dictionary  DiagnosticDataType=Manager
264     ${resp}=  Redfish.Post
265     ...  /redfish/v1/Managers/bmc/LogServices/Dump/Actions/LogService.CollectDiagnosticData
266     ...  body=${payload}  valid_status_codes=[${HTTP_ACCEPTED}]
267
268     # Example of response from above Redfish POST request.
269     # "@odata.id": "/redfish/v1/TaskService/Tasks/0",
270     # "@odata.type": "#Task.v1_4_3.Task",
271     # "Id": "0",
272     # "TaskState": "Running",
273     # "TaskStatus": "OK"
274
275     [Return]  ${resp.dict['Id']}
276
277Create User Initiated BMC Dump Via Redfish
278    [Documentation]  Generate user initiated BMC dump via Redfish and return the dump id number (e.g., "5").
279
280    ${payload}=  Create Dictionary  DiagnosticDataType=Manager
281    ${resp}=  Redfish.Post  /redfish/v1/Managers/bmc/LogServices/Dump/Actions/LogService.CollectDiagnosticData
282    ...  body=${payload}  valid_status_codes=[${HTTP_ACCEPTED}]
283
284    # Example of response from above Redfish POST request.
285    # "@odata.id": "/redfish/v1/TaskService/Tasks/0",
286    # "@odata.type": "#Task.v1_4_3.Task",
287    # "Id": "0",
288    # "TaskState": "Running",
289    # "TaskStatus": "OK"
290
291    Wait Until Keyword Succeeds  5 min  15 sec  Check Task Completion  ${resp.dict['Id']}
292    ${task_id}=  Set Variable  ${resp.dict['Id']}
293
294    ${task_dict}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
295
296    # Example of HttpHeaders field of task details.
297    # "Payload": {
298    #   "HttpHeaders": [
299    #     "Host: <BMC_IP>",
300    #      "Accept-Encoding: identity",
301    #      "Connection: Keep-Alive",
302    #      "Accept: */*",
303    #      "Content-Length: 33",
304    #      "Location: /redfish/v1/Managers/bmc/LogServices/Dump/Entries/2"]
305    #    ],
306    #    "HttpOperation": "POST",
307    #    "JsonBody": "{\"DiagnosticDataType\":\"Manager\"}",
308    #     "TargetUri": "/redfish/v1/Managers/bmc/LogServices/Dump/Actions/LogService.CollectDiagnosticData"
309    # }
310
311    [Return]  ${task_dict["Payload"]["HttpHeaders"][-1].split("/")[-1]}
312
313Auto Generate BMC Dump
314    [Documentation]  Auto generate BMC dump.
315
316    ${cmd}=  Catenate  busctl --verbose call xyz.openbmc_project.Dump.Manager
317    ...  /xyz/openbmc_project/dump/bmc xyz.openbmc_project.Dump.Create CreateDump a{sv} 0
318    ${stdout}  ${stderr}  ${rc}=
319    ...  BMC Execute Command  ${cmd}
320    [Return]  ${stdout}  ${stderr}  ${rc}
321
322Get Dump Size
323    [Documentation]  Get dump size.
324    [Arguments]  ${dump_uri}
325
326    # Description of argument(s):
327    # dump_uri        Dump URI
328    #                 (Eg. 	/xyz/openbmc_project/dump/bmc/entry/1).
329
330    # Example of Dump entry.
331    # "data": {
332    #   "CompletedTime": 1616760931,
333    #   "Elapsed": 1616760931,
334    #   "OffloadUri": "",
335    #   "Offloaded": false,
336    #   "Password": "",
337    #   "Size": 3056,
338    #   "SourceDumpId": 117440513,
339    #   "StartTime": 1616760931,
340    #   "Status": "xyz.openbmc_project.Common.Progress.OperationStatus.Completed",
341    #   "VSPString": ""
342    #  },
343
344    Log  ${dump_uri}
345    ${dump_data}=  Redfish.Get Properties  ${dump_uri}
346    [Return]  ${dump_data["data"]["Size"]}
347
348Get Dump ID
349    [Documentation]  Return dump ID.
350    [Arguments]   ${task_id}
351
352    # Description of argument(s):
353    # task_id        Task ID.
354
355    # Example of HttpHeaders field of task details.
356    # "Payload": {
357    #   "HttpHeaders": [
358    #     "Host: <BMC_IP>",
359    #      "Accept-Encoding: identity",
360    #      "Connection: Keep-Alive",
361    #      "Accept: */*",
362    #      "Content-Length: 33",
363    #      "Location: /redfish/v1/Managers/bmc/LogServices/Dump/Entries/2"]
364    #    ],
365    #    "HttpOperation": "POST",
366    #    "JsonBody": "{\"DiagnosticDataType\":\"Manager\"}",
367    #     "TargetUri":
368    # "/redfish/v1/Managers/bmc/LogServices/Dump/Actions/LogService.CollectDiagnosticData"
369    # }
370
371    ${task_dict}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
372    ${key}  ${value}=  Set Variable  ${task_dict["Payload"]["HttpHeaders"][-1].split(":")}
373    Run Keyword If  '${key}' != 'Location'  Fail
374    [Return]  ${value.strip('/').split('/')[-1]}
375
376Get Task Status
377    [Documentation]  Return task status.
378    [Arguments]   ${task_id}
379
380    # Description of argument(s):
381    # task_id        Task ID.
382
383    ${resp}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
384    [Return]  ${resp['TaskState']}
385
386Check Task Completion
387    [Documentation]  Check if the task is complete.
388    [Arguments]   ${task_id}
389
390    # Description of argument(s):
391    # task_id        Task ID.
392
393    ${task_dict}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
394    Should Be Equal As Strings  ${task_dict['TaskState']}  Completed
395
396Get Dump ID And Status
397    [Documentation]  Return dump ID and status.
398    [Arguments]   ${task_id}
399
400    # Description of argument(s):
401    # task_id        Task ID.
402
403    Wait Until Keyword Succeeds  10 min  15 sec  Check Task Completion  ${task_id}
404    ${dump_id}=  Get Dump ID  ${task_id}
405    [Return]  ${dump_id}  Completed
406
407
408Create BMC User Dump
409    [Documentation]  Generate user initiated BMC dump via Redfish and return
410    ...  the task instance Id and response object (e.g., "5").
411
412    ${payload}=  Create Dictionary  DiagnosticDataType=Manager
413    ${resp}=  Redfish.Post  /redfish/v1/Managers/bmc/LogServices/Dump/Actions/LogService.CollectDiagnosticData
414    ...  body=${payload}  valid_status_codes=[${HTTP_ACCEPTED}]
415
416    ${ip_resp}=  Evaluate  json.loads(r'''${resp.text}''')  json
417
418    Return From Keyword  ${ip_resp["Id"]}  ${resp}
419
420
421Wait For Task Completion
422    [Documentation]  Check whether the state of task instance matches any of the
423    ...  expected completion states before maximum number of retries exceeds and
424    ...  exit loop in case completion state is met.
425    [Arguments]  ${task_id}  ${expected_status}  ${retry_max_count}=300
426    ...  ${check_state}=${FALSE}
427
428    # Description of argument(s):
429    # task_id                     the task id for which completion is
430    #                             to be monitored.
431    # expected_status             the task state which is to be considered as the
432    #                             end of task life cycle.
433    # retry_max_count             the maximum number of retry count to wait for
434    #                             task to reach its completion state. Default
435    #                             value of retry_max_count is 300.
436    # check_state                 if set as TRUE, the task state will be
437    #                             monitored whether the task state value is
438    #                             valid throughout task life cycle until
439    #                             expected completion state is reached.
440    #                             Default value of check_state is FALSE.
441
442    FOR  ${retry}  IN RANGE  ${retry_max_count}
443        ${resp}=  Redfish.Get Properties  /redfish/v1/TaskService/Tasks/${task_id}
444        ${current_task_state}=  Set Variable  ${resp["TaskState"]}
445        Rprint Vars  current_task_state
446
447        Run Keyword If  ${check_state} == ${TRUE}  Should Be True
448        ...  '${resp["TaskState"]}' in ${allowed_task_state}
449        ...  msg=Verify task state is valid
450
451        Exit For Loop If
452        ...  '${resp["TaskState"]}' in ${expected_status}
453
454        Sleep  5s
455    END
456
457Get Dump Status In BMC
458    [Documentation]  Get dump status from BMC using busctl method.
459    [Arguments]  ${dump_uri}
460
461    # Description of argument(s):
462    # dump_uri   Dump URI E.g: /xyz/openbmc_project/dump/bmc/entry/7.
463
464    ${cmd}=  Catenate  busctl get-property xyz.openbmc_project.Dump.Manager
465    ...  ${dump_uri} xyz.openbmc_project.Common.Progress Status
466
467    ${stdout}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd}
468    Log  ${stdout}
469    # Example output:
470    # s "xyz.openbmc_project.Common.Progress.OperationStatus.Completed".
471
472    ${status}=  Set Variable  ${stdout.split('.')[-1].strip('"')}
473    [Return]  ${status}
474
475Verify Dump Status In BMC
476    [Documentation]  Verify Dump Status in BMC.
477    [Arguments]  ${dump_uri}  ${expected_dump_status}
478
479    # Description of argument(s):
480    # dump_uri              Dump URI E.g: /xyz/openbmc_project/dump/bmc/entry/7.
481    # expected_dump_status  Expected Dump Status (Completed or Failed etc).
482
483    ${dump_status}=  Get Dump Status In BMC  ${dump_uri}
484    Should Be Equal  ${dump_status}  ${expected_dump_status}
485