xref: /openbmc/openbmc-test-automation/lib/openbmc_ffdc_methods.robot (revision 28e403b8c632ff68954aa1e3b7fd9ca10ee1b332)
1*** Settings ***
2Documentation      Methods to execute commands on BMC and collect
3...                data to a list of FFDC files
4
5Resource           openbmc_ffdc_utils.robot
6Resource           rest_client.robot
7Resource           utils.robot
8Library            SSHLibrary
9Library            OperatingSystem
10Library            Collections
11Library            String
12Library            gen_print.py
13Library            gen_robot_keyword.py
14
15*** Keywords ***
16
17################################################################
18# Method : Call FFDC Methods                                   #
19#          Execute the user define keywords from the FFDC List #
20#          Unlike any other keywords this will call into the   #
21#          list of keywords defined in the FFDC list at one go #
22################################################################
23
24Call FFDC Methods
25    [Documentation]   Call into FFDC Keyword index list.
26    [Arguments]  ${ffdc_function_list}=${EMPTY}
27
28    # Description of argument(s):
29    # ffdc_function_list  A colon-delimited list naming the kinds of FFDC that
30    #                     is to be collected
31    #                     (e.g. "FFDC Generic Report:BMC Specific Files").
32    #                     Acceptable values can be found in the description
33    #                     field of FFDC_METHOD_CALL in
34    #                     lib/openbmc_ffdc_list.py.  Those values can be
35    #                     obtained via a call to 'Get FFDC Method Desc' (also
36    #                     from lib/openbmc_ffdc_list.py).
37
38    @{entries}=  Get FFDC Method Index
39    :FOR  ${index}  IN  @{entries}
40    \    Method Call Keyword List  ${index}  ${ffdc_function_list}
41    Run Key U  SSHLibrary.Close All Connections
42
43Method Call Keyword List
44    [Documentation]  Iterate the list through keyword index.
45    [Arguments]  ${index}  ${ffdc_function_list}=${EMPTY}
46
47    # Description of argument(s):
48    # index               The index into the FFDC_METHOD_CALL dictionary (e.g.
49    #                     'BMC LOGS').
50    # ffdc_function_list  See ffdc_function_list description in
51    #                     "Call FFDC Methods" (above).
52
53    @{method_list}=  Get ffdc method call  ${index}
54
55    # If function list is empty assign default (i.e. a list of all allowable
56    # values).  In either case, convert ffdc_function_list from a string to
57    # a list.
58    @{ffdc_function_list}=
59    ...  Run Keyword If  '${ffdc_function_list}' == '${EMPTY}'
60    ...    Get FFDC Method Desc  ${index}
61    ...  ELSE
62    ...    Split String  ${ffdc_function_list}  separator=:
63
64    :FOR  ${method}  IN  @{method_list}
65    \    Execute Keyword Method  ${method[0]}  ${method[1]}
66    ...      @{ffdc_function_list}
67
68Execute Keyword Method
69    [Documentation]  Call into BMC method keywords. Don't let one
70    ...              failure skip the remaining. Get whatever data
71    ...              it could gather at worse case scenario.
72    [Arguments]  ${description}  ${keyword_name}  @{ffdc_function_list}
73
74    # Description of argument(s):
75    # description         The description of the FFDC to be collected.  This
76    #                     would be any value returned by
77    #                     'Get FFDC Method Desc' (e.g. "FFDC Generic Report").
78    # keyword_name        The name of the keyword to call to collect the FFDC
79    #                     data (again, see FFDC_METHOD_CALL).
80    # ffdc_function_list  See ffdc_function_list description in
81    #                     "Call FFDC Methods" (above).  The only difference is
82    #                     in this case, it should be a list rather than a
83    #                     colon-delimited value.
84
85    ${status}  ${ret_values}=  Run Keyword And Ignore Error
86    ...  List Should Contain Value  ${ffdc_function_list}  ${description}
87    Run Keyword If  '${status}' != 'PASS'  Return from Keyword
88
89    Run Key  ${keyword_name}  ignore=1
90
91################################################################
92# Method : BMC FFDC Manifest                                   #
93#          Execute command on BMC and write to ffdc_report.txt #
94################################################################
95
96BMC FFDC Manifest
97    [Documentation]    Get the commands index for the FFDC_BMC_CMD,
98    ...                login to BMC and execute commands.
99
100    @{entries}=     Get ffdc cmd index
101    :FOR  ${index}  IN   @{entries}
102    \     Iterate BMC Command List Pairs   ${index}
103
104
105Iterate BMC Command List Pairs
106    [Documentation]    Feed in key pair list from dictionary to execute
107    [Arguments]        ${key_index}
108
109    @{cmd_list}=      Get ffdc bmc cmd    ${key_index}
110    Set Suite Variable   ${ENTRY_INDEX}   ${key_index}
111    :FOR  ${cmd}  IN  @{cmd_list}
112    \    Execute Command and Write FFDC    ${cmd[0]}  ${cmd[1]}
113
114
115Execute Command and Write FFDC
116    [Documentation]    Execute command on BMC or OS and write to ffdc
117    ...                By default to ffdc_report.txt file else to
118    ...                specified file path.
119    [Arguments]        ${key_index}
120    ...                ${cmd}
121    ...                ${logpath}=${FFDC_FILE_PATH}
122    ...                ${target}=BMC
123
124    Run Keyword If   '${logpath}' == '${FFDC_FILE_PATH}'
125    ...    Write Cmd Output to FFDC File   ${key_index}  ${cmd}
126
127    ${cmd_buf}=  Catenate  ${target} Execute Command \ ${cmd} \ ignore_err=${1}
128    ${status}  ${ret_values}=  Run Key  ${cmd_buf}  ignore=${1}
129
130    ${stdout}=  Set Variable  @{ret_values}[0]
131    ${stderr}=  Set Variable  @{ret_values}[1]
132
133    # Write stdout on success and stderr/stdout to the file on failure.
134    Run Keyword If  $stderr == '${EMPTY}'
135    ...    Write Data To File  ${stdout}${\n}  ${logpath}
136    ...  ELSE  Write Data To File
137    ...    ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
138    ...    ${logpath}
139
140
141################################################################
142# Method : BMC FFDC Files                                      #
143#          Execute command on BMC and write to individual file #
144#          based on the file name pre-defined in the list      #
145################################################################
146
147BMC FFDC Files
148    [Documentation]    Get the command list and iterate
149    @{entries}=     Get ffdc file index
150    :FOR  ${index}  IN   @{entries}
151    \     Create File and Write Data   ${index}
152
153
154Create File and Write Data
155    [Documentation]    Create files to current FFDC log directory,
156    ...                executes command and write to corresponding
157    ...                file name in the current FFDC directory.
158    [Arguments]        ${key_index}
159
160    @{cmd_list}=      Get ffdc bmc file   ${key_index}
161    :FOR  ${cmd}  IN  @{cmd_list}
162    \   ${logpath}=  Catenate  SEPARATOR=   ${LOG_PREFIX}   ${cmd[0]}.txt
163    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
164
165
166
167################################################################
168# Method : Log Test Case Status                                #
169#          Creates test result history footprint for reference #
170################################################################
171
172Log Test Case Status
173    [Documentation]  Test case execution result history.
174    ...  Create once and append to this file
175    ...  logs/test_history.txt
176    ...  Format   Date:Test suite:Test case:Status
177    ...  20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL
178
179    ${FFDC_DIR_PATH_STYLE}=  Get Variable Value  ${FFDC_DIR_PATH_STYLE}
180    ...  ${EMPTY}
181    ${FFDC_DIR_PATH}=  Get Variable Value  ${FFDC_DIR_PATH}  ${EMPTY}
182
183    Run Keyword If  '${FFDC_DIR_PATH}' == '${EMPTY}'  Set FFDC Defaults
184
185    Run Keyword If  '${FFDC_DIR_PATH_STYLE}' == '${1}'  Run Keywords
186    ...  Set Global Variable  ${FFDC_LOG_PATH}  ${FFDC_DIR_PATH}  AND
187    ...  Set Global Variable  ${TEST_HISTORY}  ${FFDC_DIR_PATH}test_history.txt
188
189    Create Directory   ${FFDC_LOG_PATH}
190
191    ${exist}=   Run Keyword and Return Status
192    ...   OperatingSystem.File Should Exist   ${TEST_HISTORY}
193
194    Run Keyword If  '${exist}' == '${False}'
195    ...   Create File  ${TEST_HISTORY}
196
197    Rpvars  TEST_HISTORY
198
199    ${cur_time}=      Get Current Time Stamp
200
201    Append To File    ${TEST_HISTORY}
202    ...   ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}
203
204
205Log FFDC Get Requests
206    [Documentation]    Create file in current FFDC log directory.
207    ...                Do openbmc get request and write to
208    ...                corresponding file name.
209    ...                JSON pretty print for logging to file.
210    [Arguments]        ${key_index}
211
212    @{cmd_list}=  Get ffdc get request  ${key_index}
213    :FOR  ${cmd}  IN  @{cmd_list}
214    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
215    \   ${resp}=  OpenBMC Get Request  ${cmd[1]}  quiet=${1}
216    \   ${status}=    Run Keyword and Return Status
217    ...   Should Be Equal As Strings    ${resp.status_code}    ${HTTP_OK}
218    \   Run Keyword If   '${status}' == '${False}'  Continue For Loop
219    \   ${jsondata}=  to json  ${resp.content}    pretty_print=True
220    \   Write Data To File  ${\n}${jsondata}${\n}  ${logpath}
221
222
223BMC FFDC Get Requests
224    [Documentation]    Get the command list and iterate
225    @{entries}=  Get ffdc get request index
226    :FOR  ${index}  IN  @{entries}
227    \   Log FFDC Get Requests   ${index}
228
229
230Log OS ALL DISTROS FFDC
231    [Documentation]    Create file in current FFDC log directory.
232    ...                Executes OS command and write to
233    ...                corresponding file name.
234    [Arguments]        ${key_index}
235
236    @{cmd_list}=  get ffdc os all distros call  ${key_index}
237    :FOR  ${cmd}  IN  @{cmd_list}
238    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
239    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}  ${logpath}
240    \   ...  target=OS
241
242
243Log OS SPECIFIC DISTRO FFDC
244    [Documentation]    Create file in current FFDC log directory.
245    ...                Executes OS command and write to
246    ...                corresponding file name.
247    [Arguments]        ${key_index}  ${linux_distro}
248
249    @{cmd_list}=  get ffdc os distro call  ${key_index}  ${linux_distro}
250    :FOR  ${cmd}  IN  @{cmd_list}
251    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
252    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}  ${logpath}
253    \   ...  target=OS
254
255
256OS FFDC Files
257    [Documentation]    Get the command list and iterate
258    [Arguments]  ${OS_HOST}=${OS_HOST}  ${OS_USERNAME}=${OS_USERNAME}
259    ...   ${OS_PASSWORD}=${OS_PASSWORD}
260
261    Return From Keyword If  '${OS_HOST}' == '${EMPTY}'
262    ...   No OS Host Provided
263
264    # If can't ping, return
265    ${rc}=  Run Keyword and Return Status  Ping Host  ${OS_HOST}
266    Return From Keyword If  '${rc}' == '${False}'
267    ...   Could not ping OS.
268
269    ${stdout}  ${stderr}  ${rc}=  OS Execute Command  uptime  ignore_err=${1}
270    Return From Keyword If  '${rc}' != '${0}'  Could not connect to OS.
271
272    ${stdout}  ${stderr}  ${rc}=  OS Execute Command
273    ...  . /etc/os-release; echo $ID  ignore_err=${0}
274    Set Global Variable  ${linux_distro}  ${stdout}
275
276    Rpvars  linux_distro
277
278    @{entries}=  Get ffdc os all distros index
279    :FOR  ${index}  IN  @{entries}
280    \   Log OS ALL DISTROS FFDC  ${index}
281
282    Return From Keyword If
283    ...  '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
284    ...  Could not determine Linux Distribution.
285
286    @{entries}=  Get ffdc os distro index  ${linux_distro}
287    :FOR  ${index}  IN  @{entries}
288    \   Log OS SPECIFIC DISTRO FFDC  ${index}  ${linux_distro}
289
290
291System Inventory Files
292    [Documentation]  Copy systest os_inventory files.
293    # The os_inventory files are the result of running
294    # systest/htx_hardbootme_test.  If these files exist
295    # they are copied to the FFDC directory.
296    # Global variable ffdc_dir_path is the path name of the
297    # directory they are copied to.
298    Copy Files  os_inventory_*.json  ${ffdc_dir_path}
299    Remove Files  os_inventory_*.json
300
301
302##############################################################################
303SCP Coredump Files
304    [Documentation]  Copy core dump file from BMC to local system.
305
306    # Check if core dump exist in the /tmp
307    ${core_files}  ${stderr}  ${rc}=  BMC Execute Command  ls /tmp/core_*
308    @{core_list} =  Split String    ${core_files}
309    # Copy the core files
310    Run Key U  Open Connection for SCP
311    :FOR  ${index}  IN  @{core_list}
312    \  scp.Get File  ${index}  ${LOG_PREFIX}${index.lstrip("/tmp/")}
313    # Remove the file from remote to avoid re-copying on next FFDC call
314    \  BMC Execute Command  rm ${index}
315    # I can't find a way to do this: scp.Close Connection
316
317
318##############################################################################
319Collect eSEL Log
320    [Documentation]  Collect eSEL log from logging entry and convert eSEL data
321    ...              to elog formatted string text file.
322    [Arguments]  ${log_prefix_path}=${LOG_PREFIX}
323
324    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}/enumerate  quiet=${1}
325    ${status}=  Run Keyword And Return Status
326    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
327    Return From Keyword If  '${status}' == '${False}'
328
329    ${content}=  To Json  ${resp.content}
330    # Grab the list of entries from logging/entry/
331    # The data shown below is the result of the "Get Dictionary Keys".
332    # Example:
333    # /xyz/openbmc_project/logging/entry/1
334    # /xyz/openbmc_project/logging/entry/2
335    ${esel_list}=  Get Dictionary Keys  ${content['data']}
336
337    ${logpath}=  Catenate  SEPARATOR=  ${log_prefix_path}  esel
338    Create File  ${logpath}
339    # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData
340    #  "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
341    # Sample eSEL entry:
342    #  "/xyz/openbmc_project/logging/entry/1": {
343    #    "Timestamp": 1487744317025,
344    #    "AdditionalData": [
345    #        "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
346    #    ],
347    #    "Message": "org.open_power.Error.Host.Event.Event",
348    #    "Id": 1,
349    #    "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency"
350    # }
351
352    :FOR  ${entry_path}  IN  @{esel_list}
353    # Skip reading attribute if entry URI is a callout.
354    # Example: /xyz/openbmc_project/logging/entry/1/callout
355    \  Continue For Loop If  '${entry_path.rsplit('/', 1)[1]}' == 'callout'
356    \  ${esel_data}=  Read Attribute  ${entry_path}  AdditionalData  quiet=${1}
357    \  ${status}=  Run Keyword And Return Status
358    ...  Should Contain Match  ${esel_data}  ESEL*
359    \  Continue For Loop If  ${status} == ${False}
360    \  ${index}=  Get Esel Index  ${esel_data}
361    \  Write Data To File  "${esel_data[${index}]}"  ${logpath}
362    \  Write Data To File  ${\n}  ${logpath}
363
364    ${out}=  Run  which eSEL.pl
365    ${status}=  Run Keyword And Return Status
366    ...  Should Contain  ${out}  eSEL.pl
367    Return From Keyword If  '${status}' == '${False}'
368
369    Convert eSEL To Elog Format  ${logpath}
370
371
372##############################################################################
373Convert eSEL To Elog Format
374    [Documentation]  Execute parser tool on the eSEL data file to generate
375    ...              formatted error log.
376    [Arguments]  ${esel_file_path}
377    # Description of arguments:
378    # esel_file_path  Absolute path of the eSEL data (e.g.
379    #                 /tmp/w55.170404.154820.esel).
380
381    # Note: The only way to get eSEL.pl to put the output in a particular
382    # directory is to cd to that directory.
383    ${cmd_buf}=  Catenate  cd $(dirname ${esel_file_path}) ; eSEL.pl -l
384    ...  ${esel_file_path} -p decode_obmc_data
385    Run  ${cmd_buf}
386
387##############################################################################
388