1*** Settings ***
2Documentation      Methods to execute commands on BMC and collect
3...                data to a list of FFDC files
4
5Resource               openbmc_ffdc_utils.robot
6Resource               rest_client.robot
7Resource               utils.robot
8Resource               list_utils.robot
9Resource               logging_utils.robot
10Library                SSHLibrary
11Library                OperatingSystem
12Library                Collections
13Library                String
14Library                gen_print.py
15Library                gen_cmd.py
16Library                gen_robot_keyword.py
17Library                dump_utils.py
18Library                logging_utils.py
19
20*** Variables ***
21
22${FFDC_CMD_TIMEOUT}    240
23
24*** Keywords ***
25
26# Method : Call FFDC Methods                                   #
27#          Execute the user define keywords from the FFDC List #
28#          Unlike any other keywords this will call into the   #
29#          list of keywords defined in the FFDC list at one go #
30
31Call FFDC Methods
32    [Documentation]   Call into FFDC Keyword index list.
33    [Arguments]  ${ffdc_function_list}=${EMPTY}
34
35    # Description of argument(s):
36    # ffdc_function_list  A colon-delimited list naming the kinds of FFDC that
37    #                     are to be collected
38    #                     (e.g. "FFDC Generic Report:BMC Specific Files").
39    #                     Acceptable values can be found in the description
40    #                     field of FFDC_METHOD_CALL in
41    #                     lib/openbmc_ffdc_list.py.  Those values can be
42    #                     obtained via a call to 'Get FFDC Method Desc' (also
43    #                     from lib/openbmc_ffdc_list.py).
44
45    @{entries}=  Get FFDC Method Index
46    # Example entries:
47    # entries:
48    #   entries[0]:  BMC LOGS
49
50    @{ffdc_file_list}=  Create List
51    :FOR  ${index}  IN  @{entries}
52    \    ${ffdc_file_sub_list}=  Method Call Keyword List  ${index}
53    ...      ${ffdc_function_list}
54    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
55    ...      ${ffdc_file_sub_list}
56
57    Run Key U  SSHLibrary.Close All Connections
58
59    [Return]  ${ffdc_file_list}
60
61Method Call Keyword List
62    [Documentation]  Process FFDC request and return a list of generated files.
63    [Arguments]  ${index}  ${ffdc_function_list}=${EMPTY}
64
65    # Description of argument(s):
66    # index               The index into the FFDC_METHOD_CALL dictionary (e.g.
67    #                     'BMC LOGS').
68    # ffdc_function_list  See ffdc_function_list description in
69    #                     "Call FFDC Methods" (above).
70
71    @{method_list}=  Get FFDC Method Call  ${index}
72    # Example method_list:
73    # method_list:
74    #   method_list[0]:
75    #     method_list[0][0]: FFDC Generic Report
76    #     method_list[0][1]: BMC FFDC Manifest
77    #   method_list[1]:
78    #     method_list[1][0]: Get Request FFDC
79    #     method_list[1][1]: BMC FFDC Get Requests
80    # (etc.)
81
82    # If function list is empty assign default (i.e. a list of all allowable
83    # values).  In either case, convert ffdc_function_list from a string to
84    # a list.
85    @{ffdc_function_list}=
86    ...  Run Keyword If  '${ffdc_function_list}' == '${EMPTY}'
87    ...    Get FFDC Method Desc  ${index}
88    ...  ELSE
89    ...    Split String  ${ffdc_function_list}  separator=:
90
91    @{ffdc_file_list}=  Create List
92    :FOR  ${method}  IN  @{method_list}
93    \    ${ffdc_file_sub_list}=  Execute Keyword Method  ${method[0]}
94    ...      ${method[1]}  @{ffdc_function_list}
95    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
96    ...      ${ffdc_file_sub_list}
97
98    [Return]  ${ffdc_file_list}
99
100
101Execute Keyword Method
102    [Documentation]  Call into BMC method keywords. Don't let one
103    ...              failure skip the remaining. Get whatever data
104    ...              it could gather at worse case scenario.
105    [Arguments]  ${description}  ${keyword_name}  @{ffdc_function_list}
106
107    # Description of argument(s):
108    # description         The description of the FFDC to be collected.  This
109    #                     would be any value returned by
110    #                     'Get FFDC Method Desc' (e.g. "FFDC Generic Report").
111    # keyword_name        The name of the keyword to call to collect the FFDC
112    #                     data (again, see FFDC_METHOD_CALL).
113    # ffdc_function_list  See ffdc_function_list description in
114    #                     "Call FFDC Methods" (above).  The only difference is
115    #                     in this case, it should be a list rather than a
116    #                     colon-delimited value.
117
118    @{ffdc_file_list}=  Create List
119
120    ${index}=  Get Index From List  ${ffdc_function_list}  ${description}
121    Run Keyword If  '${index}' == '${-1}'  Return from Keyword
122    ...  ${ffdc_file_list}
123
124    ${status}  ${ffdc_file_list}=  Run Key  ${keyword_name}  ignore=1
125    [Return]  ${ffdc_file_list}
126
127# Method : BMC FFDC Manifest                                   #
128#          Execute command on BMC and write to ffdc_report.txt #
129
130BMC FFDC Manifest
131    [Documentation]  Run the ssh commands from FFDC_BMC_CMD and return a list
132    ...              of generated files.
133
134    @{ffdc_file_list}=  Create List  ${FFDC_FILE_PATH}
135    @{entries}=  Get FFDC Cmd Index
136    :FOR  ${index}  IN  @{entries}
137    \    Iterate BMC Command List Pairs  ${index}
138
139    [Return]  ${ffdc_file_list}
140
141
142Iterate BMC Command List Pairs
143    [Documentation]    Feed in key pair list from dictionary to execute
144    [Arguments]        ${key_index}
145
146    @{cmd_list}=      Get ffdc bmc cmd    ${key_index}
147    Set Suite Variable   ${ENTRY_INDEX}   ${key_index}
148    :FOR  ${cmd}  IN  @{cmd_list}
149    \    Execute Command and Write FFDC    ${cmd[0]}  ${cmd[1]}
150
151
152Execute Command and Write FFDC
153    [Documentation]  Run a command on the BMC or OS, write the output to the
154    ...              specified file and return a list of generated files.
155    [Arguments]  ${key_index}  ${cmd}  ${logpath}=${FFDC_FILE_PATH}
156    ...          ${target}=BMC
157
158    Run Keyword If  '${logpath}' == '${FFDC_FILE_PATH}'
159    ...    Write Cmd Output to FFDC File  ${key_index}  ${cmd}
160
161    @{ffdc_file_list}=  Create List  ${log_path}
162
163    ${cmd_buf}=  Catenate  ${target} Execute Command \ ${cmd} \ ignore_err=${1}
164    ...  \ time_out=${FFDC_CMD_TIMEOUT}
165    ${status}  ${ret_values}=  Run Key  ${cmd_buf}  ignore=${1}
166    # If the command times out, status will be 'FAIL'.
167    Return From Keyword If  '${status}' == 'FAIL'  ${ffdc_file_list}
168
169    ${stdout}=  Set Variable  @{ret_values}[0]
170    ${stderr}=  Set Variable  @{ret_values}[1]
171
172    # Write stdout on success and stderr/stdout to the file on failure.
173    Run Keyword If  $stderr == '${EMPTY}'
174    ...    Write Data To File  ${stdout}${\n}  ${logpath}
175    ...  ELSE  Write Data To File
176    ...    ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
177    ...    ${logpath}
178
179    [Return]  ${ffdc_file_list}
180
181
182# Method : BMC FFDC Files                                      #
183#          Execute command on BMC and write to individual file #
184#          based on the file name pre-defined in the list      #
185
186BMC FFDC Files
187    [Documentation]  Run the commands from FFDC_BMC_FILE and return a list of
188    ...              generated files.
189
190    @{entries}=  Get FFDC File Index
191    # Example of entries:
192    # entries:
193    #   entries[0]: BMC FILES
194
195    @{ffdc_file_list}=  Create List
196    :FOR  ${index}  IN  @{entries}
197    \    ${ffdc_file_sub_list}=  Create File and Write Data  ${index}
198    \     ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
199    ...       ${ffdc_file_sub_list}
200
201    [Return]  ${ffdc_file_list}
202
203
204Create File and Write Data
205    [Documentation]  Run commands from FFDC_BMC_FILE to create FFDC files and
206    ...              return a list of generated files.
207    [Arguments]  ${key_index}
208
209    # Description of argument(s):
210    # key_index  The index into the FFDC_BMC_FILE dictionary.
211
212    @{ffdc_file_list}=  Create List
213    @{cmd_list}=  Get FFDC BMC File  ${key_index}
214    :FOR  ${cmd}  IN  @{cmd_list}
215    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
216    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
217    ...      ${cmd[1]}  ${logpath}
218    \     ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
219    ...       ${ffdc_file_sub_list}
220
221    [Return]  ${ffdc_file_list}
222
223
224# Method : Log Test Case Status                                #
225#          Creates test result history footprint for reference #
226
227Log Test Case Status
228    [Documentation]  Test case execution result history.
229    ...  Create once and append to this file
230    ...  logs/test_history.txt
231    ...  Format   Date:Test suite:Test case:Status
232    ...  20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL
233
234    ${FFDC_DIR_PATH_STYLE}=  Get Variable Value  ${FFDC_DIR_PATH_STYLE}
235    ...  ${EMPTY}
236    ${FFDC_DIR_PATH}=  Get Variable Value  ${FFDC_DIR_PATH}  ${EMPTY}
237
238    Run Keyword If  '${FFDC_DIR_PATH}' == '${EMPTY}'  Set FFDC Defaults
239
240    Run Keyword If  '${FFDC_DIR_PATH_STYLE}' == '${1}'  Run Keywords
241    ...  Set Global Variable  ${FFDC_LOG_PATH}  ${FFDC_DIR_PATH}  AND
242    ...  Set Global Variable  ${TEST_HISTORY}  ${FFDC_DIR_PATH}test_history.txt
243
244    Create Directory   ${FFDC_LOG_PATH}
245
246    ${exist}=   Run Keyword and Return Status
247    ...   OperatingSystem.File Should Exist   ${TEST_HISTORY}
248
249    Run Keyword If  '${exist}' == '${False}'
250    ...   Create File  ${TEST_HISTORY}
251
252    Rpvars  TEST_HISTORY
253
254    ${cur_time}=      Get Current Time Stamp
255
256    Append To File    ${TEST_HISTORY}
257    ...   ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}
258
259
260Log FFDC Get Requests
261    [Documentation]  Run the get requests associated with the key and return a
262    ...              list of generated files.
263    [Arguments]  ${key_index}
264
265    # Note: Output will be in JSON pretty_print format.
266
267    # Description of argument(s):
268    # key_index  The key to the FFDC_GET_REQUEST dictionary that contains the
269    #            get requests that are to be run.
270
271    @{ffdc_file_list}=  Create List
272    @{cmd_list}=  Get FFDC Get Request  ${key_index}
273    :FOR  ${cmd}  IN  @{cmd_list}
274    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
275    \    ${resp}=  OpenBMC Get Request  ${cmd[1]}  quiet=${1}
276    \    ${status}=  Run Keyword and Return Status
277    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
278    \    Run Keyword If  '${status}' == '${False}'  Continue For Loop
279    \    ${jsondata}=  to json  ${resp.content}  pretty_print=True
280    \    Write Data To File  ${\n}${jsondata}${\n}  ${logpath}
281    \    Append To List  ${ffdc_file_list}  ${logpath}
282
283    [Return]  ${ffdc_file_list}
284
285BMC FFDC Get Requests
286    [Documentation]  Iterate over get request list and return a list of
287    ...              generated files.
288
289    @{ffdc_file_list}=  Create List
290
291    @{entries}=  Get ffdc get request index
292    # Example of entries:
293    # entries:
294    #  entries[0]:  GET REQUESTS
295    :FOR  ${index}  IN  @{entries}
296    \    ${ffdc_file_sub_list}=  Log FFDC Get Requests  ${index}
297    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
298    ...  ${ffdc_file_sub_list}
299
300    [Return]  ${ffdc_file_list}
301
302Log OS All distros FFDC
303    [Documentation]  Run commands from FFDC_OS_ALL_DISTROS_FILE to create FFDC
304    ...              files and return a list of generated files.
305    [Arguments]  ${key_index}
306
307    # Description of argument(s):
308    # key_index  The index into the FFDC_OS_ALL_DISTROS_FILE dictionary.
309
310    @{ffdc_file_list}=  Create List
311
312    @{cmd_list}=  Get FFDC OS All Distros Call  ${key_index}
313    :FOR  ${cmd}  IN  @{cmd_list}
314    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
315    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
316    ...      ${cmd[1]}  ${logpath}  target=OS
317    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
318    ...      ${ffdc_file_sub_list}
319
320    [Return]  ${ffdc_file_list}
321
322
323Log OS SPECIFIC DISTRO FFDC
324    [Documentation]  Run commands from the FFDC_OS_<distro>_FILE to create FFDC
325    ...              files and return a list of generated files.
326    [Arguments]  ${key_index}  ${linux_distro}
327
328    # Description of argument(s):
329    # key_index  The index into the FFDC_OS_<distro>_FILE dictionary.
330    # linux_distro  Your OS's linux distro (e.g. "UBUNTU", "RHEL", etc).
331
332    @{ffdc_file_list}=  Create List
333
334    @{cmd_list}=  Get FFDC OS Distro Call  ${key_index}  ${linux_distro}
335    :FOR  ${cmd}  IN  @{cmd_list}
336    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
337    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
338    ...      ${cmd[1]}  ${logpath}  target=OS
339    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
340    ...      ${ffdc_file_sub_list}
341
342    [Return]  ${ffdc_file_list}
343
344
345OS FFDC Files
346    [Documentation]  Run the commands from FFDC_OS_ALL_DISTROS_FILE to create
347    ...              FFDC files and return a list of generated files.
348    [Arguments]  ${OS_HOST}=${OS_HOST}  ${OS_USERNAME}=${OS_USERNAME}
349    ...  ${OS_PASSWORD}=${OS_PASSWORD}
350
351    @{ffdc_file_list}=  Create List
352
353    Run Keyword If  '${OS_HOST}' == '${EMPTY}'  Run Keywords
354    ...  Print Timen  No OS Host provided so no OS FFDC will be done.  AND
355    ...  Return From Keyword  ${ffdc_file_list}
356
357    ${match_state}=  Create Dictionary  os_ping=^1$  os_login=^1$
358    ...  os_run_cmd=^1$
359    ${status}  ${ret_values}=  Run Keyword and Ignore Error  Check State
360    ...  ${match_state}  quiet=0
361
362    Run Keyword If  '${status}' == 'FAIL'  Run Keywords
363    ...  Print Timen  The OS is not communicating so no OS FFDC will be done.\n
364    ...  AND
365    ...  Return From Keyword  ${ffdc_file_list}
366
367    ${stdout}  ${stderr}  ${rc}=  OS Execute Command
368    ...  . /etc/os-release; echo $ID  ignore_err=${0}
369    Set Global Variable  ${linux_distro}  ${stdout}
370
371    Rpvars  linux_distro
372
373    @{entries}=  Get FFDC OS All Distros Index
374    :FOR  ${index}  IN  @{entries}
375    \    ${ffdc_file_sub_list}=  Log OS All distros FFDC  ${index}
376    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
377    ...      ${ffdc_file_sub_list}
378
379    Return From Keyword If
380    ...  '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
381    ...  ${ffdc_file_list}
382
383    @{entries}=  Get ffdc os distro index  ${linux_distro}
384    :FOR  ${index}  IN  @{entries}
385    \    ${ffdc_file_sub_list}=  Log OS SPECIFIC DISTRO FFDC  ${index}
386    ...      ${linux_distro}
387    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
388    ...      ${ffdc_file_sub_list}
389
390    [Return]  ${ffdc_file_list}
391
392
393System Inventory Files
394    [Documentation]  Copy systest os_inventory files and return a list of
395    ...              generated files..
396    # The os_inventory files are the result of running
397    # systest/htx_hardbootme_test.  If these files exist
398    # they are copied to the FFDC directory.
399    # Global variable ffdc_dir_path is the path name of the
400    # directory they are copied to.
401
402    @{ffdc_file_list}=  Create List
403
404    ${globex}=  Set Variable  os_inventory_*.json
405
406    @{file_list}=  OperatingSystem.List Files In Directory  .  ${globex}
407
408    Copy Files  ${globex}  ${ffdc_dir_path}
409
410    : FOR  ${file_name}  IN  @{file_list}
411    \    Append To List  ${ffdc_file_list}  ${ffdc_dir_path}${file_name}
412
413    Run Keyword and Ignore Error  Remove Files  ${globex}
414
415    [Return]  ${ffdc_file_list}
416
417
418SCP Coredump Files
419    [Documentation]  Copy core dump files from BMC to local system and return a
420    ...              list of generated file names.
421
422    @{ffdc_file_list}=  Create List
423
424    # Check if core dump exist in the /tmp
425    ${core_files}  ${stderr}  ${rc}=  BMC Execute Command  ls /tmp/core_*
426    ...  ignore_err=${1}
427    Run Keyword If  '${rc}' != '${0}'  Return From Keyword  ${ffdc_file_list}
428
429    @{core_list}=  Split String  ${core_files}
430    # Copy the core files
431    Run Key U  Open Connection for SCP
432    :FOR  ${index}  IN  @{core_list}
433    \    ${ffdc_file_path}=  Catenate  ${LOG_PREFIX}${index.lstrip("/tmp/")}
434    \    ${status}=  Run Keyword and Return Status
435    ...  scp.Get File  ${index}  ${ffdc_file_path}
436    \    Run Keyword If  '${status}' == '${False}'  Continue For Loop
437    \    Append To List  ${ffdc_file_list}  ${ffdc_file_path}
438    # Remove the file from remote to avoid re-copying on next FFDC call
439    \    BMC Execute Command  rm ${index}  ignore_err=${1}
440    # I can't find a way to do this: scp.Close Connection
441
442    [Return]  ${ffdc_file_list}
443
444SCP Dump Files
445    [Documentation]  Copy all dump files from BMC to local system.
446
447    # Check if dumps exist
448    ${ffdc_file_list}=  Scp Dumps  ${FFDC_DIR_PATH}  ${FFDC_PREFIX}
449
450    [Return]  ${ffdc_file_list}
451
452Collect Dump Log
453    [Documentation]  Collect dumps from dump entry.
454    [Arguments]  ${log_prefix_path}=${LOG_PREFIX}
455
456    ${data}=  Read Properties  ${DUMP_ENTRY_URI}enumerate  quiet=${1}
457
458    # Grab the list of entries from dump/entry/
459    # The data shown below is the result of the "Get Dictionary Keys".
460    # Example:
461    # /xyz/openbmc_project/dump/entry/1
462    # /xyz/openbmc_project/dump/entry/2
463
464    ${dump_list}=  Get Dictionary Keys  ${data}
465
466
467Collect eSEL Log
468    [Documentation]  Create raw and formatted eSEL files.
469    [Arguments]  ${log_prefix_path}=${LOG_PREFIX}
470
471    # NOTE: If no eSEL.pl program can be located, then no formatted eSEL file
472    # will be generated.
473
474    # Description of argument(s):
475    # log_prefix_path               The path prefix to be used in creating
476    #                               eSEL file path names.  For example, if
477    #                               log_prefix_path is
478    #                               "/tmp/user1/dummy.181001.120000.", then
479    #                               this keyword will create
480    #                               /tmp/user1/dummy.181001.120000.esel (raw)
481    #                               and
482    #                               /tmp/user1/dummy.181001.120000.esel.txt
483    #                               (formatted).
484
485    @{ffdc_file_list}=  Create List
486
487    ${esels}=  Get Esels
488    ${num_esels}=  Evaluate  len(${esels})
489    Rprint Vars  num_esels
490    Return From Keyword If  ${num_esels} == ${0}  ${ffdc_file_list}
491
492    ${logpath}=  Catenate  SEPARATOR=  ${log_prefix_path}  esel
493    Create File  ${logpath}
494
495    :FOR  ${esel}  IN  @{esels}
496    \  Write Data To File  "${esel}"${\n}  ${logpath}
497
498    Append To List  ${ffdc_file_list}  ${logpath}
499
500    ${rc}  ${output}=  Shell Cmd  which eSEL.pl  show_err=0
501    Return From Keyword If  ${rc} != ${0}  ${ffdc_file_list}
502
503    Convert eSEL To Elog Format  ${logpath}
504    Append To List  ${ffdc_file_list}  ${logpath}.txt
505
506    [Return]  ${ffdc_file_list}
507
508
509Convert eSEL To Elog Format
510    [Documentation]  Execute parser tool on the eSEL data file to generate
511    ...              formatted error log.
512    [Arguments]  ${esel_file_path}
513    # Description of argument(s):
514    # esel_file_path                The path to the file containing raw eSEL
515    #                               data (e.g.
516    #                               "/tmp/user1/dummy.181001.120000.esel").
517
518    # Note: The only way to get eSEL.pl to put the output in a particular
519    # directory is to cd to that directory.
520    ${cmd_buf}=  Catenate  cd $(dirname ${esel_file_path}) ; eSEL.pl -l
521    ...  ${esel_file_path} -p decode_obmc_data
522    Qprint Issuing  ${cmd_buf}
523    Run  ${cmd_buf}
524    # The .binary file, which is generated by eSEL.pl, is of no use to us.
525    Remove File  ${esel_file_path}.binary
526