xref: /openbmc/openbmc-test-automation/lib/openbmc_ffdc_methods.robot (revision 4e8c09b1a6aace6f6bd1dfae1b999ba871f5c9b5)
1*** Settings ***
2Documentation      Methods to execute commands on BMC and collect
3...                data to a list of FFDC files
4
5Resource           openbmc_ffdc_utils.robot
6Resource           rest_client.robot
7Resource           utils.robot
8Resource           list_utils.robot
9Library            SSHLibrary
10Library            OperatingSystem
11Library            Collections
12Library            String
13Library            gen_print.py
14Library            gen_robot_keyword.py
15
16*** Keywords ***
17
18################################################################
19# Method : Call FFDC Methods                                   #
20#          Execute the user define keywords from the FFDC List #
21#          Unlike any other keywords this will call into the   #
22#          list of keywords defined in the FFDC list at one go #
23################################################################
24
25Call FFDC Methods
26    [Documentation]   Call into FFDC Keyword index list.
27    [Arguments]  ${ffdc_function_list}=${EMPTY}
28
29    # Description of argument(s):
30    # ffdc_function_list  A colon-delimited list naming the kinds of FFDC that
31    #                     are to be collected
32    #                     (e.g. "FFDC Generic Report:BMC Specific Files").
33    #                     Acceptable values can be found in the description
34    #                     field of FFDC_METHOD_CALL in
35    #                     lib/openbmc_ffdc_list.py.  Those values can be
36    #                     obtained via a call to 'Get FFDC Method Desc' (also
37    #                     from lib/openbmc_ffdc_list.py).
38
39    @{entries}=  Get FFDC Method Index
40    # Example entries:
41    # entries:
42    #   entries[0]:  BMC LOGS
43
44    @{ffdc_file_list}=  Create List
45    :FOR  ${index}  IN  @{entries}
46    \    ${ffdc_file_sub_list}=  Method Call Keyword List  ${index}
47    ...      ${ffdc_function_list}
48    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
49    ...      ${ffdc_file_sub_list}
50
51    Run Key U  SSHLibrary.Close All Connections
52
53    [Return]  ${ffdc_file_list}
54
55Method Call Keyword List
56    [Documentation]  Process FFDC request and return a list of generated files.
57    [Arguments]  ${index}  ${ffdc_function_list}=${EMPTY}
58
59    # Description of argument(s):
60    # index               The index into the FFDC_METHOD_CALL dictionary (e.g.
61    #                     'BMC LOGS').
62    # ffdc_function_list  See ffdc_function_list description in
63    #                     "Call FFDC Methods" (above).
64
65    @{method_list}=  Get FFDC Method Call  ${index}
66    # Example method_list:
67    # method_list:
68    #   method_list[0]:
69    #     method_list[0][0]: FFDC Generic Report
70    #     method_list[0][1]: BMC FFDC Manifest
71    #   method_list[1]:
72    #     method_list[1][0]: Get Request FFDC
73    #     method_list[1][1]: BMC FFDC Get Requests
74    # (etc.)
75
76    # If function list is empty assign default (i.e. a list of all allowable
77    # values).  In either case, convert ffdc_function_list from a string to
78    # a list.
79    @{ffdc_function_list}=
80    ...  Run Keyword If  '${ffdc_function_list}' == '${EMPTY}'
81    ...    Get FFDC Method Desc  ${index}
82    ...  ELSE
83    ...    Split String  ${ffdc_function_list}  separator=:
84
85    @{ffdc_file_list}=  Create List
86    :FOR  ${method}  IN  @{method_list}
87    \    ${ffdc_file_sub_list}=  Execute Keyword Method  ${method[0]}
88    ...      ${method[1]}  @{ffdc_function_list}
89    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
90    ...      ${ffdc_file_sub_list}
91
92    [Return]  ${ffdc_file_list}
93
94
95Execute Keyword Method
96    [Documentation]  Call into BMC method keywords. Don't let one
97    ...              failure skip the remaining. Get whatever data
98    ...              it could gather at worse case scenario.
99    [Arguments]  ${description}  ${keyword_name}  @{ffdc_function_list}
100
101    # Description of argument(s):
102    # description         The description of the FFDC to be collected.  This
103    #                     would be any value returned by
104    #                     'Get FFDC Method Desc' (e.g. "FFDC Generic Report").
105    # keyword_name        The name of the keyword to call to collect the FFDC
106    #                     data (again, see FFDC_METHOD_CALL).
107    # ffdc_function_list  See ffdc_function_list description in
108    #                     "Call FFDC Methods" (above).  The only difference is
109    #                     in this case, it should be a list rather than a
110    #                     colon-delimited value.
111
112    @{ffdc_file_list}=  Create List
113
114    ${index}=  Get Index From List  ${ffdc_function_list}  ${description}
115    Run Keyword If  '${index}' == '${-1}'  Return from Keyword
116    ...  ${ffdc_file_list}
117
118    ${status}  ${ffdc_file_list}=  Run Key  ${keyword_name}  ignore=1
119    [Return]  ${ffdc_file_list}
120
121################################################################
122# Method : BMC FFDC Manifest                                   #
123#          Execute command on BMC and write to ffdc_report.txt #
124################################################################
125
126BMC FFDC Manifest
127    [Documentation]  Run the ssh commands from FFDC_BMC_CMD and return a list
128    ...              of generated files.
129
130    @{ffdc_file_list}=  Create List  ${FFDC_FILE_PATH}
131    @{entries}=  Get FFDC Cmd Index
132    :FOR  ${index}  IN  @{entries}
133    \    Iterate BMC Command List Pairs  ${index}
134
135    [Return]  ${ffdc_file_list}
136
137
138Iterate BMC Command List Pairs
139    [Documentation]    Feed in key pair list from dictionary to execute
140    [Arguments]        ${key_index}
141
142    @{cmd_list}=      Get ffdc bmc cmd    ${key_index}
143    Set Suite Variable   ${ENTRY_INDEX}   ${key_index}
144    :FOR  ${cmd}  IN  @{cmd_list}
145    \    Execute Command and Write FFDC    ${cmd[0]}  ${cmd[1]}
146
147
148Execute Command and Write FFDC
149    [Documentation]  Run a command on the BMC or OS, write the output to the
150    ...              specified file and return a list of generated files.
151    [Arguments]  ${key_index}  ${cmd}  ${logpath}=${FFDC_FILE_PATH}
152    ...          ${target}=BMC
153
154    Run Keyword If  '${logpath}' == '${FFDC_FILE_PATH}'
155    ...    Write Cmd Output to FFDC File  ${key_index}  ${cmd}
156
157    @{ffdc_file_list}=  Create List  ${log_path}
158
159    ${cmd_buf}=  Catenate  ${target} Execute Command \ ${cmd} \ ignore_err=${1}
160    ${status}  ${ret_values}=  Run Key  ${cmd_buf}  ignore=${1}
161
162    ${stdout}=  Set Variable  @{ret_values}[0]
163    ${stderr}=  Set Variable  @{ret_values}[1]
164
165    # Write stdout on success and stderr/stdout to the file on failure.
166    Run Keyword If  $stderr == '${EMPTY}'
167    ...    Write Data To File  ${stdout}${\n}  ${logpath}
168    ...  ELSE  Write Data To File
169    ...    ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
170    ...    ${logpath}
171
172    [Return]  ${ffdc_file_list}
173
174
175################################################################
176# Method : BMC FFDC Files                                      #
177#          Execute command on BMC and write to individual file #
178#          based on the file name pre-defined in the list      #
179################################################################
180
181BMC FFDC Files
182    [Documentation]  Run the commands from FFDC_BMC_FILE and return a list of
183    ...              generated files.
184
185    @{entries}=  Get FFDC File Index
186    # Example of entries:
187    # entries:
188    #   entries[0]: BMC FILES
189
190    @{ffdc_file_list}=  Create List
191    :FOR  ${index}  IN  @{entries}
192    \    ${ffdc_file_sub_list}=  Create File and Write Data  ${index}
193    \     ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
194    ...       ${ffdc_file_sub_list}
195
196    [Return]  ${ffdc_file_list}
197
198
199Create File and Write Data
200    [Documentation]  Run commands from FFDC_BMC_FILE to create FFDC files and
201    ...              return a list of generated files.
202    [Arguments]  ${key_index}
203
204    # Description of argument(s):
205    # key_index  The index into the FFDC_BMC_FILE dictionary.
206
207    @{ffdc_file_list}=  Create List
208    @{cmd_list}=  Get FFDC BMC File  ${key_index}
209    :FOR  ${cmd}  IN  @{cmd_list}
210    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
211    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
212    ...      ${cmd[1]}  ${logpath}
213    \     ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
214    ...       ${ffdc_file_sub_list}
215
216    [Return]  ${ffdc_file_list}
217
218
219################################################################
220# Method : Log Test Case Status                                #
221#          Creates test result history footprint for reference #
222################################################################
223
224Log Test Case Status
225    [Documentation]  Test case execution result history.
226    ...  Create once and append to this file
227    ...  logs/test_history.txt
228    ...  Format   Date:Test suite:Test case:Status
229    ...  20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL
230
231    ${FFDC_DIR_PATH_STYLE}=  Get Variable Value  ${FFDC_DIR_PATH_STYLE}
232    ...  ${EMPTY}
233    ${FFDC_DIR_PATH}=  Get Variable Value  ${FFDC_DIR_PATH}  ${EMPTY}
234
235    Run Keyword If  '${FFDC_DIR_PATH}' == '${EMPTY}'  Set FFDC Defaults
236
237    Run Keyword If  '${FFDC_DIR_PATH_STYLE}' == '${1}'  Run Keywords
238    ...  Set Global Variable  ${FFDC_LOG_PATH}  ${FFDC_DIR_PATH}  AND
239    ...  Set Global Variable  ${TEST_HISTORY}  ${FFDC_DIR_PATH}test_history.txt
240
241    Create Directory   ${FFDC_LOG_PATH}
242
243    ${exist}=   Run Keyword and Return Status
244    ...   OperatingSystem.File Should Exist   ${TEST_HISTORY}
245
246    Run Keyword If  '${exist}' == '${False}'
247    ...   Create File  ${TEST_HISTORY}
248
249    Rpvars  TEST_HISTORY
250
251    ${cur_time}=      Get Current Time Stamp
252
253    Append To File    ${TEST_HISTORY}
254    ...   ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}
255
256
257Log FFDC Get Requests
258    [Documentation]  Run the get requests associated with the key and return a
259    ...              list of generated files.
260    [Arguments]  ${key_index}
261
262    # Note: Output will be in JSON pretty_print format.
263
264    # Description of argument(s):
265    # key_index  The key to the FFDC_GET_REQUEST dictionary that contains the
266    #            get requests that are to be run.
267
268    @{ffdc_file_list}=  Create List
269    @{cmd_list}=  Get FFDC Get Request  ${key_index}
270    :FOR  ${cmd}  IN  @{cmd_list}
271    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
272    \    ${resp}=  OpenBMC Get Request  ${cmd[1]}  quiet=${1}
273    \    ${status}=  Run Keyword and Return Status
274    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
275    \    Run Keyword If  '${status}' == '${False}'  Continue For Loop
276    \    ${jsondata}=  to json  ${resp.content}  pretty_print=True
277    \    Write Data To File  ${\n}${jsondata}${\n}  ${logpath}
278    \    Append To List  ${ffdc_file_list}  ${logpath}
279
280    [Return]  ${ffdc_file_list}
281
282BMC FFDC Get Requests
283    [Documentation]  Iterate over get request list and return a list of
284    ...              generated files.
285
286    @{ffdc_file_list}=  Create List
287
288    @{entries}=  Get ffdc get request index
289    # Example of entries:
290    # entries:
291    #  entries[0]:  GET REQUESTS
292    :FOR  ${index}  IN  @{entries}
293    \    ${ffdc_file_sub_list}=  Log FFDC Get Requests  ${index}
294    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
295    ...  ${ffdc_file_sub_list}
296
297    [Return]  ${ffdc_file_list}
298
299Log OS All distros FFDC
300    [Documentation]  Run commands from FFDC_OS_ALL_DISTROS_FILE to create FFDC
301    ...              files and return a list of generated files.
302    [Arguments]  ${key_index}
303
304    # Description of argument(s):
305    # key_index  The index into the FFDC_OS_ALL_DISTROS_FILE dictionary.
306
307    @{ffdc_file_list}=  Create List
308
309    @{cmd_list}=  Get FFDC OS All Distros Call  ${key_index}
310    :FOR  ${cmd}  IN  @{cmd_list}
311    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
312    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
313    ...      ${cmd[1]}  ${logpath}  target=OS
314    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
315    ...      ${ffdc_file_sub_list}
316
317    [Return]  ${ffdc_file_list}
318
319
320Log OS SPECIFIC DISTRO FFDC
321    [Documentation]  Run commands from the FFDC_OS_<distro>_FILE to create FFDC
322    ...              files and return a list of generated files.
323    [Arguments]  ${key_index}  ${linux_distro}
324
325    # Description of argument(s):
326    # key_index  The index into the FFDC_OS_<distro>_FILE dictionary.
327    # linux_distro  Your OS's linux distro (e.g. "UBUNTU", "RHEL", etc).
328
329    @{ffdc_file_list}=  Create List
330
331    @{cmd_list}=  Get FFDC OS Distro Call  ${key_index}  ${linux_distro}
332    :FOR  ${cmd}  IN  @{cmd_list}
333    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
334    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
335    ...      ${cmd[1]}  ${logpath}  target=OS
336    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
337    ...      ${ffdc_file_sub_list}
338
339    [Return]  ${ffdc_file_list}
340
341
342OS FFDC Files
343    [Documentation]  Run the commands from FFDC_OS_ALL_DISTROS_FILE to create
344    ...              FFDC files and return a list of generated files.
345    [Arguments]  ${OS_HOST}=${OS_HOST}  ${OS_USERNAME}=${OS_USERNAME}
346    ...  ${OS_PASSWORD}=${OS_PASSWORD}
347
348    @{ffdc_file_list}=  Create List
349
350    Run Keyword If  '${OS_HOST}' == '${EMPTY}'  Run Keywords
351    ...  Print Timen  No OS Host provided so no OS FFDC will be done.  AND
352    ...  Return From Keyword  ${ffdc_file_list}
353
354    ${match_state}=  Create Dictionary  os_ping=^1$  os_login=^1$
355    ...  os_run_cmd=^1$
356    ${status}  ${ret_values}=  Run Keyword and Ignore Error  Check State
357    ...  ${match_state}  quiet=0
358
359    Run Keyword If  '${status}' == 'FAIL'  Run Keywords
360    ...  Print Error  The OS is not communicating so no OS FFDC will be done.\n
361    ...  AND
362    ...  Return From Keyword  ${ffdc_file_list}
363
364    ${stdout}  ${stderr}  ${rc}=  OS Execute Command
365    ...  . /etc/os-release; echo $ID  ignore_err=${0}
366    Set Global Variable  ${linux_distro}  ${stdout}
367
368    Rpvars  linux_distro
369
370    @{entries}=  Get FFDC OS All Distros Index
371    :FOR  ${index}  IN  @{entries}
372    \    ${ffdc_file_sub_list}=  Log OS All distros FFDC  ${index}
373    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
374    ...      ${ffdc_file_sub_list}
375
376    Return From Keyword If
377    ...  '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
378    ...  ${ffdc_file_list}
379
380    @{entries}=  Get ffdc os distro index  ${linux_distro}
381    :FOR  ${index}  IN  @{entries}
382    \    ${ffdc_file_sub_list}=  Log OS SPECIFIC DISTRO FFDC  ${index}
383    ...      ${linux_distro}
384    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
385    ...      ${ffdc_file_sub_list}
386
387    [Return]  ${ffdc_file_list}
388
389
390System Inventory Files
391    [Documentation]  Copy systest os_inventory files and return a list of
392    ...              generated files..
393    # The os_inventory files are the result of running
394    # systest/htx_hardbootme_test.  If these files exist
395    # they are copied to the FFDC directory.
396    # Global variable ffdc_dir_path is the path name of the
397    # directory they are copied to.
398
399    @{ffdc_file_list}=  Create List
400
401    ${globex}=  Set Variable  os_inventory_*.json
402
403    @{file_list}=  OperatingSystem.List Files In Directory  .  ${globex}
404
405    Copy Files  ${globex}  ${ffdc_dir_path}
406
407    : FOR  ${file_name}  IN  @{file_list}
408    \    Append To List  ${ffdc_file_list}  ${ffdc_dir_path}${file_name}
409
410    Run Keyword and Ignore Error  Remove Files  ${globex}
411
412    [Return]  ${ffdc_file_list}
413
414
415##############################################################################
416SCP Coredump Files
417    [Documentation]  Copy core dump files from BMC to local system and return a
418    ...              list of generated file names.
419
420    @{ffdc_file_list}=  Create List
421
422    # Check if core dump exist in the /tmp
423    ${core_files}  ${stderr}  ${rc}=  BMC Execute Command  ls /tmp/core_*
424    ...  ignore_err=${1}
425    Run Keyword If  '${rc}' != '${0}'  Return From Keyword  ${ffdc_file_list}
426
427    @{core_list}=  Split String  ${core_files}
428    # Copy the core files
429    Run Key U  Open Connection for SCP
430    :FOR  ${index}  IN  @{core_list}
431    \    ${ffdc_file_path}=  Catenate  ${LOG_PREFIX}${index.lstrip("/tmp/")}
432    \    ${status}=  Run Keyword and Return Status
433    ...  scp.Get File  ${index}  ${ffdc_file_path}
434    \    Run Keyword If  '${status}' == '${False}'  Continue For Loop
435    \    Append To List  ${ffdc_file_list}  ${ffdc_file_path}
436    # Remove the file from remote to avoid re-copying on next FFDC call
437    \    BMC Execute Command  rm ${index}  ignore_err=${1}
438    # I can't find a way to do this: scp.Close Connection
439
440    [Return]  ${ffdc_file_list}
441
442
443##############################################################################
444Collect eSEL Log
445    [Documentation]  Collect eSEL log from logging entry and convert eSEL data
446    ...              to elog formatted string text file.
447    [Arguments]  ${log_prefix_path}=${LOG_PREFIX}
448
449    @{ffdc_file_list}=  Create List
450
451    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}/enumerate  quiet=${1}
452    ${status}=  Run Keyword And Return Status
453    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
454    Return From Keyword If  '${status}' == '${False}'  ${ffdc_file_list}
455
456    ${content}=  To Json  ${resp.content}
457    # Grab the list of entries from logging/entry/
458    # The data shown below is the result of the "Get Dictionary Keys".
459    # Example:
460    # /xyz/openbmc_project/logging/entry/1
461    # /xyz/openbmc_project/logging/entry/2
462    ${esel_list}=  Get Dictionary Keys  ${content['data']}
463
464    ${logpath}=  Catenate  SEPARATOR=  ${log_prefix_path}  esel
465    Create File  ${logpath}
466    # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData
467    #  "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
468    # Sample eSEL entry:
469    #  "/xyz/openbmc_project/logging/entry/1": {
470    #    "Timestamp": 1487744317025,
471    #    "AdditionalData": [
472    #        "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
473    #    ],
474    #    "Message": "org.open_power.Error.Host.Event.Event",
475    #    "Id": 1,
476    #    "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency"
477    # }
478
479    :FOR  ${entry_path}  IN  @{esel_list}
480    # Skip reading attribute if entry URI is a callout.
481    # Example: /xyz/openbmc_project/logging/entry/1/callout
482    \  Continue For Loop If  '${entry_path.rsplit('/', 1)[1]}' == 'callout'
483    \  ${esel_data}=  Read Attribute  ${entry_path}  AdditionalData  quiet=${1}
484    \  ${status}=  Run Keyword And Return Status
485    ...  Should Contain Match  ${esel_data}  ESEL*
486    \  Continue For Loop If  ${status} == ${False}
487    \  ${index}=  Get Esel Index  ${esel_data}
488    \  Write Data To File  "${esel_data[${index}]}"  ${logpath}
489    \  Write Data To File  ${\n}  ${logpath}
490
491    ${out}=  Run  which eSEL.pl
492    ${status}=  Run Keyword And Return Status
493    ...  Should Contain  ${out}  eSEL.pl
494    Return From Keyword If  '${status}' == '${False}'  ${ffdc_file_list}
495
496    Convert eSEL To Elog Format  ${logpath}
497
498    Append To List  ${ffdc_file_list}  ${logpath}
499
500    [Return]  ${ffdc_file_list}
501
502
503##############################################################################
504Convert eSEL To Elog Format
505    [Documentation]  Execute parser tool on the eSEL data file to generate
506    ...              formatted error log.
507    [Arguments]  ${esel_file_path}
508    # Description of arguments:
509    # esel_file_path  Absolute path of the eSEL data (e.g.
510    #                 /tmp/w55.170404.154820.esel).
511
512    # Note: The only way to get eSEL.pl to put the output in a particular
513    # directory is to cd to that directory.
514    ${cmd_buf}=  Catenate  cd $(dirname ${esel_file_path}) ; eSEL.pl -l
515    ...  ${esel_file_path} -p decode_obmc_data
516    Run  ${cmd_buf}
517
518##############################################################################
519