1*** Settings ***
2Documentation      Methods to execute commands on BMC and collect
3...                data to a list of FFDC files
4
5Resource           openbmc_ffdc_utils.robot
6Resource           rest_client.robot
7Resource           utils.robot
8Library            SSHLibrary
9Library            OperatingSystem
10Library            Collections
11
12*** Keywords ***
13
14################################################################
15# Method : Call FFDC Methods                                   #
16#          Execute the user define keywords from the FFDC List #
17#          Unlike any other keywords this will call into the   #
18#          list of keywords defined in the FFDC list at one go #
19################################################################
20
21Call FFDC Methods
22    [Documentation]   Calls into FFDC Keyword index list
23
24    @{entries}=     Get ffdc method index
25    :FOR  ${index}  IN   @{entries}
26    \     Method Call Keyword List   ${index}
27    SSHLibrary.Close All Connections
28
29
30Method Call Keyword List
31    [Documentation]   Iterate the list through keyword index
32    [Arguments]       ${index}
33
34    @{method_list}=      Get ffdc method call   ${index}
35    :FOR  ${method}  IN  @{method_list}
36    \    Execute Keyword Method   ${method[1]}
37
38
39Execute Keyword Method
40    [Documentation]   Calls into BMC method keywords. Don't let one
41    ...               failure skips the remaining. Get whatever data
42    ...               it could gather at worse case scenario.
43    [Arguments]   ${keyword_name}
44
45    Run Keyword And Continue On Failure   ${keyword_name}
46
47
48################################################################
49# Method : BMC FFDC Manifest                                   #
50#          Execute command on BMC and write to ffdc_report.txt #
51################################################################
52
53BMC FFDC Manifest
54    [Documentation]    Get the commands index for the FFDC_BMC_CMD,
55    ...                login to BMC and execute commands.
56    Open Connection And Log In
57
58    @{entries}=     Get ffdc cmd index
59    :FOR  ${index}  IN   @{entries}
60    \     Iterate BMC Command List Pairs   ${index}
61
62
63Iterate BMC Command List Pairs
64    [Documentation]    Feed in key pair list from dictionary to execute
65    [Arguments]        ${key_index}
66
67    @{cmd_list}=      Get ffdc bmc cmd    ${key_index}
68    Set Suite Variable   ${ENTRY_INDEX}   ${key_index}
69    :FOR  ${cmd}  IN  @{cmd_list}
70    \    Execute Command and Write FFDC    ${cmd[0]}  ${cmd[1]}
71
72
73Execute Command and Write FFDC
74    [Documentation]    Execute command on BMC or OS and write to ffdc
75    ...                By default to ffdc_report.txt file else to
76    ...                specified file path.
77    [Arguments]        ${key_index}
78    ...                ${cmd}
79    ...                ${logpath}=${FFDC_FILE_PATH}
80
81    Run Keyword If   '${logpath}' == '${FFDC_FILE_PATH}'
82    ...    Write Cmd Output to FFDC File   ${key_index}  ${cmd}
83
84    ${stdout}  ${stderr}=
85    ...   Execute Command    ${cmd}   return_stderr=True
86
87    # Write stdout on success and stderr/stdout to the file on failure.
88    Run Keyword If  $stderr == '${EMPTY}'
89    ...    Write Data To File  ${stdout}${\n}  ${logpath}
90    ...  ELSE  Write Data To File
91    ...    ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
92    ...    ${logpath}
93
94
95################################################################
96# Method : BMC FFDC Files                                      #
97#          Execute command on BMC and write to individual file #
98#          based on the file name pre-defined in the list      #
99################################################################
100
101BMC FFDC Files
102    [Documentation]    Get the command list and iterate
103    Open Connection And Log In
104    @{entries}=     Get ffdc file index
105    :FOR  ${index}  IN   @{entries}
106    \     Create File and Write Data   ${index}
107
108
109Create File and Write Data
110    [Documentation]    Create files to current FFDC log directory,
111    ...                executes command and write to corresponding
112    ...                file name in the current FFDC directory.
113    [Arguments]        ${key_index}
114
115    # To build IP address in searchable form eg: dummy\.domain\.com
116    ${OPENBMC_HOST_REGEX}=  Run  echo ${OPENBMC_HOST} | sed 's/\(\.\)/\\\1/g'
117    @{cmd_list}=      Get ffdc bmc file   ${key_index}
118    :FOR  ${cmd}  IN  @{cmd_list}
119    \   ${logpath}=  Catenate  SEPARATOR=   ${LOG_PREFIX}   ${cmd[0]}.txt
120    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
121    # Rename OPENBMC_HOST IP address from given file to DUMMYIP
122    \   Run  sed -i 's/'${OPENBMC_HOST_REGEX}'/DUMMYIP/g' ${logpath}
123
124
125
126################################################################
127# Method : Log Test Case Status                                #
128#          Creates test result history footprint for reference #
129################################################################
130
131Log Test Case Status
132    [Documentation]  Test case execution result history.
133    ...  Create once and append to this file
134    ...  logs/test_history.txt
135    ...  Format   Date:Test suite:Test case:Status
136    ...  20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL
137    Create Directory   ${FFDC_LOG_PATH}
138
139    ${exist}=   Run Keyword and Return Status
140    ...   OperatingSystem.File Should Exist   ${TEST_HISTORY}
141
142    Run Keyword If  '${exist}' == '${False}'
143    ...   Create File  ${TEST_HISTORY}
144
145    ${cur_time}=      Get Current Time Stamp
146
147    Append To File    ${TEST_HISTORY}
148    ...   ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}
149
150
151Log FFDC Get Requests
152    [Documentation]    Create file in current FFDC log directory.
153    ...                Do openbmc get request and write to
154    ...                corresponding file name.
155    ...                JSON pretty print for logging to file.
156    [Arguments]        ${key_index}
157
158    @{cmd_list}=  Get ffdc get request  ${key_index}
159    :FOR  ${cmd}  IN  @{cmd_list}
160    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
161    \   ${resp}=  OpenBMC Get Request  ${cmd[1]}  quiet=${1}
162    \   ${status}=    Run Keyword and Return Status
163    ...   Should Be Equal As Strings    ${resp.status_code}    ${HTTP_OK}
164    \   Run Keyword If   '${status}' == '${False}'  Continue For Loop
165    \   ${jsondata}=  to json  ${resp.content}    pretty_print=True
166    \   Write Data To File  ${\n}${jsondata}${\n}  ${logpath}
167
168
169BMC FFDC Get Requests
170    [Documentation]    Get the command list and iterate
171    Open Connection And Log In
172    @{entries}=  Get ffdc get request index
173    :FOR  ${index}  IN  @{entries}
174    \   Log FFDC Get Requests   ${index}
175
176
177Log OS ALL DISTROS FFDC
178    [Documentation]    Create file in current FFDC log directory.
179    ...                Executes OS command and write to
180    ...                corresponding file name.
181    [Arguments]        ${key_index}
182
183    @{cmd_list}=  get ffdc os all distros call  ${key_index}
184    :FOR  ${cmd}  IN  @{cmd_list}
185    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
186    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
187
188
189Log OS SPECIFIC DISTRO FFDC
190    [Documentation]    Create file in current FFDC log directory.
191    ...                Executes OS command and write to
192    ...                corresponding file name.
193    [Arguments]        ${key_index}  ${linux_distro}
194
195    @{cmd_list}=  get ffdc os distro call  ${key_index}  ${linux_distro}
196    :FOR  ${cmd}  IN  @{cmd_list}
197    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
198    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
199
200
201
202OS FFDC Files
203    [Documentation]    Get the command list and iterate
204    [Arguments]  ${OS_HOST}=${OS_HOST}  ${OS_USERNAME}=${OS_USERNAME}
205    ...   ${OS_PASSWORD}=${OS_PASSWORD}
206
207    Return From Keyword If  '${OS_HOST}' == '${EMPTY}'
208    ...   No OS Host Provided
209
210    # If can't ping, return
211    ${rc}=  Run Keyword and Return Status  Ping Host  ${OS_HOST}
212    Return From Keyword If  '${rc}' == '${False}'
213    ...   Could not ping OS
214
215    Open Connection And Log In  host=${OS_HOST}  username=${OS_USERNAME}
216    ...   password=${OS_PASSWORD}
217
218    ${output}  ${stderr}  ${rc}=  Execute Command  uptime  return_stderr=True
219    ...   return_rc=True
220
221    # If the return code returned by "Execute Command" is non-zero, return
222    Return From Keyword If  '${rc}' != '${0}'
223    ...   Could not connect to OS
224
225    @{entries}=  Get ffdc os all distros index
226    :FOR  ${index}  IN  @{entries}
227    \   Log OS ALL DISTROS FFDC  ${index}
228
229    ${linux_distro}=  Execute Command
230    ...   . /etc/os-release; echo $ID
231    ...   return_stdout=True  return_stderr=False  return_rc=False
232
233    Return From Keyword If
234    ...  '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
235    ...  Could not determine Linux Distribution
236
237    @{entries}=  Get ffdc os distro index  ${linux_distro}
238    :FOR  ${index}  IN  @{entries}
239    \   Log OS SPECIFIC DISTRO FFDC  ${index}  ${linux_distro}
240
241
242##############################################################################
243SCP Coredump Files
244    [Documentation]  Copy core dump file from BMC to local system.
245    # Check if core dump exist in the /tmp
246    Open Connection And Log In
247    ${core_files}=  Execute Command  ls /tmp/core_*
248    @{core_list} =  Split String    ${core_files}
249    # Copy the core files
250    Open Connection for SCP
251    :FOR  ${index}  IN  @{core_list}
252    \  scp.Get File  ${index}  ${LOG_PREFIX}${index.lstrip("/tmp/")}
253    # Remove the file from remote to avoid re-copying on next FFDC call
254    \  Execute Command On BMC  rm ${index}
255
256
257##############################################################################
258Collect eSEL Log
259    [Documentation]  Collect eSEL log from logging entry.
260    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}/enumerate  quiet=${1}
261    ${status}=  Run Keyword And Return Status
262    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
263    Return From Keyword If  '${status}' == '${False}'
264
265    ${content}=  To Json  ${resp.content}
266    # Grab the list of entries from logging/entry/
267    # The data shown below is the result of the "Get Dictionary Keys".
268    # Example:
269    # /xyz/openbmc_project/logging/entry/1
270    # /xyz/openbmc_project/logging/entry/2
271    ${esel_list}=  Get Dictionary Keys  ${content['data']}
272
273    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  esel
274    Create File  ${logpath}
275    # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData
276    #  "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
277    # Sample eSEL entry:
278    #  "/xyz/openbmc_project/logging/entry/1": {
279    #    "Timestamp": 1487744317025,
280    #    "AdditionalData": [
281    #        "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
282    #    ],
283    #    "Message": "org.open_power.Error.Host.Event.Event",
284    #    "Id": 1,
285    #    "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency"
286    # }
287
288    :FOR  ${entry_path}  IN  @{esel_list}
289    \  ${esel_data}=  Read Attribute  ${entry_path}  AdditionalData  quiet=${1}
290    \  ${length}=  Get Length  ${esel_data}
291    # Skip writting to file if eSEL AdditionalData is empty
292    \  Continue For Loop If  ${length} == ${0}
293    \  Write Data To File  "${esel_data[0]}"  ${logpath}
294    \  Write Data To File  ${\n}  ${logpath}
295
296##############################################################################
297