1*** Settings ***
2Documentation      Methods to execute commands on BMC and collect
3...                data to a list of FFDC files
4
5Resource           openbmc_ffdc_utils.robot
6Resource           rest_client.robot
7Resource           utils.robot
8Library            SSHLibrary
9Library            OperatingSystem
10
11*** Keywords ***
12
13################################################################
14# Method : Call FFDC Methods                                   #
15#          Execute the user define keywords from the FFDC List #
16#          Unlike any other keywords this will call into the   #
17#          list of keywords defined in the FFDC list at one go #
18################################################################
19
20Call FFDC Methods
21    [Documentation]   Calls into FFDC Keyword index list
22
23    @{entries}=     Get ffdc method index
24    :FOR  ${index}  IN   @{entries}
25    \     Method Call Keyword List   ${index}
26    SSHLibrary.Close All Connections
27
28
29Method Call Keyword List
30    [Documentation]   Iterate the list through keyword index
31    [Arguments]       ${index}
32
33    @{method_list}=      Get ffdc method call   ${index}
34    :FOR  ${method}  IN  @{method_list}
35    \    Execute Keyword Method   ${method[1]}
36
37
38Execute Keyword Method
39    [Documentation]   Calls into BMC method keywords. Don't let one
40    ...               failure skips the remaining. Get whatever data
41    ...               it could gather at worse case scenario.
42    [Arguments]   ${keyword_name}
43
44    Run Keyword And Continue On Failure   ${keyword_name}
45
46
47################################################################
48# Method : BMC FFDC Manifest                                   #
49#          Execute command on BMC and write to ffdc_report.txt #
50################################################################
51
52BMC FFDC Manifest
53    [Documentation]    Get the commands index for the FFDC_BMC_CMD,
54    ...                login to BMC and execute commands.
55    Open Connection And Log In
56
57    @{entries}=     Get ffdc cmd index
58    :FOR  ${index}  IN   @{entries}
59    \     Iterate BMC Command List Pairs   ${index}
60
61
62Iterate BMC Command List Pairs
63    [Documentation]    Feed in key pair list from dictionary to execute
64    [Arguments]        ${key_index}
65
66    @{cmd_list}=      Get ffdc bmc cmd    ${key_index}
67    Set Suite Variable   ${ENTRY_INDEX}   ${key_index}
68    :FOR  ${cmd}  IN  @{cmd_list}
69    \    Execute Command and Write FFDC    ${cmd[0]}  ${cmd[1]}
70
71
72Execute Command and Write FFDC
73    [Documentation]    Execute command on BMC or OS and write to ffdc
74    ...                By default to ffdc_report.txt file else to
75    ...                specified file path.
76    [Arguments]        ${key_index}
77    ...                ${cmd}
78    ...                ${logpath}=${FFDC_FILE_PATH}
79
80    Run Keyword If   '${logpath}' == '${FFDC_FILE_PATH}'
81    ...    Write Cmd Output to FFDC File   ${key_index}  ${cmd}
82
83    ${stdout}  ${stderr}=
84    ...   Execute Command    ${cmd}   return_stderr=True
85
86    # Write stdout on success and stderr/stdout to the file on failure.
87    Run Keyword If  $stderr == '${EMPTY}'
88    ...    Write Data To File  ${stdout}${\n}  ${logpath}
89    ...  ELSE  Write Data To File
90    ...    ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
91    ...    ${logpath}
92
93
94################################################################
95# Method : BMC FFDC Files                                      #
96#          Execute command on BMC and write to individual file #
97#          based on the file name pre-defined in the list      #
98################################################################
99
100BMC FFDC Files
101    [Documentation]    Get the command list and iterate
102    Open Connection And Log In
103    @{entries}=     Get ffdc file index
104    :FOR  ${index}  IN   @{entries}
105    \     Create File and Write Data   ${index}
106
107
108Create File and Write Data
109    [Documentation]    Create files to current FFDC log directory,
110    ...                executes command and write to corresponding
111    ...                file name in the current FFDC directory.
112    [Arguments]        ${key_index}
113
114    # To build IP address in searchable form eg: dummy\.domain\.com
115    ${OPENBMC_HOST_REGEX}=  Run  echo ${OPENBMC_HOST} | sed 's/\(\.\)/\\\1/g'
116    @{cmd_list}=      Get ffdc bmc file   ${key_index}
117    :FOR  ${cmd}  IN  @{cmd_list}
118    \   ${logpath}=  Catenate  SEPARATOR=   ${LOG_PREFIX}   ${cmd[0]}.txt
119    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
120    # Rename OPENBMC_HOST IP address from given file to DUMMYIP
121    \   Run  sed -i 's/'${OPENBMC_HOST_REGEX}'/DUMMYIP/g' ${logpath}
122
123
124
125################################################################
126# Method : Log Test Case Status                                #
127#          Creates test result history footprint for reference #
128################################################################
129
130Log Test Case Status
131    [Documentation]  Test case execution result history.
132    ...  Create once and append to this file
133    ...  logs/test_history.txt
134    ...  Format   Date:Test suite:Test case:Status
135    ...  20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL
136    Create Directory   ${FFDC_LOG_PATH}
137
138    ${exist}=   Run Keyword and Return Status
139    ...   OperatingSystem.File Should Exist   ${TEST_HISTORY}
140
141    Run Keyword If  '${exist}' == '${False}'
142    ...   Create File  ${TEST_HISTORY}
143
144    ${cur_time}=      Get Current Time Stamp
145
146    Append To File    ${TEST_HISTORY}
147    ...   ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}
148
149
150Log FFDC Get Requests
151    [Documentation]    Create file in current FFDC log directory.
152    ...                Do openbmc get request and write to
153    ...                corresponding file name.
154    ...                JSON pretty print for logging to file.
155    [Arguments]        ${key_index}
156
157    @{cmd_list}=  Get ffdc get request  ${key_index}
158    :FOR  ${cmd}  IN  @{cmd_list}
159    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
160    \   ${resp}=  OpenBMC Get Request  ${cmd[1]}  quiet=${1}
161    \   ${status}=    Run Keyword and Return Status
162    ...   Should Be Equal As Strings    ${resp.status_code}    ${HTTP_OK}
163    \   Run Keyword If   '${status}' == '${False}'  Continue For Loop
164    \   ${jsondata}=  to json  ${resp.content}    pretty_print=True
165    \   Write Data To File  ${\n}${jsondata}${\n}  ${logpath}
166
167
168BMC FFDC Get Requests
169    [Documentation]    Get the command list and iterate
170    Open Connection And Log In
171    @{entries}=  Get ffdc get request index
172    :FOR  ${index}  IN  @{entries}
173    \   Log FFDC Get Requests   ${index}
174
175
176Log OS ALL DISTROS FFDC
177    [Documentation]    Create file in current FFDC log directory.
178    ...                Executes OS command and write to
179    ...                corresponding file name.
180    [Arguments]        ${key_index}
181
182    @{cmd_list}=  get ffdc os all distros call  ${key_index}
183    :FOR  ${cmd}  IN  @{cmd_list}
184    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
185    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
186
187
188Log OS SPECIFIC DISTRO FFDC
189    [Documentation]    Create file in current FFDC log directory.
190    ...                Executes OS command and write to
191    ...                corresponding file name.
192    [Arguments]        ${key_index}  ${linux_distro}
193
194    @{cmd_list}=  get ffdc os distro call  ${key_index}  ${linux_distro}
195    :FOR  ${cmd}  IN  @{cmd_list}
196    \   ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
197    \   Execute Command and Write FFDC  ${cmd[0]}  ${cmd[1]}   ${logpath}
198
199
200
201OS FFDC Files
202    [Documentation]    Get the command list and iterate
203    [Arguments]  ${OS_HOST}=${OS_HOST}  ${OS_USERNAME}=${OS_USERNAME}
204    ...   ${OS_PASSWORD}=${OS_PASSWORD}
205
206    Return From Keyword If  '${OS_HOST}' == '${EMPTY}'
207    ...   No OS Host Provided
208
209    # If can't ping, return
210    ${rc}=  Run Keyword and Return Status  Ping Host  ${OS_HOST}
211    Return From Keyword If  '${rc}' == '${False}'
212    ...   Could not ping OS
213
214    Open Connection And Log In  host=${OS_HOST}  username=${OS_USERNAME}
215    ...   password=${OS_PASSWORD}
216
217    ${output}  ${stderr}  ${rc}=  Execute Command  uptime  return_stderr=True
218    ...   return_rc=True
219
220    # If the return code returned by "Execute Command" is non-zero, return
221    Return From Keyword If  '${rc}' != '${0}'
222    ...   Could not connect to OS
223
224    @{entries}=  Get ffdc os all distros index
225    :FOR  ${index}  IN  @{entries}
226    \   Log OS ALL DISTROS FFDC  ${index}
227
228    ${linux_distro}=  Execute Command
229    ...   . /etc/os-release; echo $ID
230    ...   return_stdout=True  return_stderr=False  return_rc=False
231
232    Return From Keyword If
233    ...  '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
234    ...  Could not determine Linux Distribution
235
236    @{entries}=  Get ffdc os distro index  ${linux_distro}
237    :FOR  ${index}  IN  @{entries}
238    \   Log OS SPECIFIC DISTRO FFDC  ${index}  ${linux_distro}
239
240
241##############################################################################
242SCP Coredump Files
243    [Documentation]  Copy core dump file from BMC to local system.
244    # Check if core dump exist in the /tmp
245    Open Connection And Log In
246    ${core_files}=  Execute Command  ls /tmp/core_*
247    @{core_list} =  Split String    ${core_files}
248    # Copy the core files
249    Open Connection for SCP
250    :FOR  ${index}  IN  @{core_list}
251    \  scp.Get File  ${index}  ${LOG_PREFIX}${index.lstrip("/tmp/")}
252    # Remove the file from remote to avoid re-copying on next FFDC call
253    \  Execute Command On BMC  rm ${index}
254
255
256##############################################################################
257Collect eSEL Log
258    [Documentation]  Collect eSEL log from logging entry.
259    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}/enumerate  quiet=${1}
260    ${status}=  Run Keyword And Return Status
261    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
262    Return From Keyword If  '${status}' == '${False}'
263
264    ${content}=  To Json  ${resp.content}
265    # Grab the list of entries from logging/entry/
266    # The data shown below is the result of the "Get Dictionary Keys".
267    # Example:
268    # /xyz/openbmc_project/logging/entry/1
269    # /xyz/openbmc_project/logging/entry/2
270    ${esel_list}=  Get Dictionary Keys  ${content['data']}
271
272    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  esel
273    Create File  ${logpath}
274    # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData
275    #  "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
276    # Sample eSEL entry:
277    #  "/xyz/openbmc_project/logging/entry/1": {
278    #    "Timestamp": 1487744317025,
279    #    "AdditionalData": [
280    #        "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
281    #    ],
282    #    "Message": "org.open_power.Error.Host.Event.Event",
283    #    "Id": 1,
284    #    "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency"
285    # }
286
287    :FOR  ${entry_path}  IN  @{esel_list}
288    \  ${esel_data}=  Read Attribute  ${entry_path}  AdditionalData  quiet=${1}
289    \  Write Data To File  "${esel_data[0]}"  ${logpath}
290    \  Write Data To File  ${\n}  ${logpath}
291
292##############################################################################
293