1*** Settings ***
2Documentation    Error logging utility keywords.
3
4Resource        rest_client.robot
5Resource        bmc_redfish_utils.robot
6Variables       ../data/variables.py
7Variables       ../data/pel_variables.py
8
9*** Variables ***
10
11
12# Define variables for use by callers of 'Get Error Logs'.
13${low_severity_errlog_regex}  \\.(Informational|Notice|Debug|OK)$
14&{low_severity_errlog_filter}  Severity=${low_severity_errlog_regex}
15&{low_severity_errlog_filter_args}  filter_dict=${low_severity_errlog_filter}  regex=${True}  invert=${True}
16# The following is equivalent to &{low_severity_errlog_filter_args} but the name may be more intuitive for
17# users. Example usage:
18# ${err_logs}=  Get Error Logs  &{filter_low_severity_errlogs}
19&{filter_low_severity_errlogs}  &{low_severity_errlog_filter_args}
20
21*** Keywords ***
22
23Get Logging Entry List
24    [Documentation]  Get logging entry and return the object list.
25
26    ${entry_list}=  Create List
27    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}list  quiet=${1}
28    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
29    ${jsondata}=  To JSON  ${resp.content}
30
31    FOR  ${entry}  IN  @{jsondata["data"]}
32        Continue For Loop If  '${entry.rsplit('/', 1)[1]}' == 'callout'
33        Append To List  ${entry_list}  ${entry}
34    END
35
36    # Logging entries list.
37    # ['/xyz/openbmc_project/logging/entry/14',
38    #  '/xyz/openbmc_project/logging/entry/15']
39    [Return]  ${entry_list}
40
41
42Logging Entry Should Exist
43    [Documentation]  Find the matching message id and return the entry id.
44    [Arguments]  ${message_id}
45
46    # Description of argument(s):
47    # message_id    Logging message string.
48    #               Example: "xyz.openbmc_project.Common.Error.InternalFailure"
49
50    @{elog_entries}=  Get Logging Entry List
51
52    FOR  ${entry}  IN  @{elog_entries}
53         ${resp}=  Read Properties  ${entry}
54         ${status}=  Run Keyword And Return Status
55         ...  Should Be Equal As Strings  ${message_id}  ${resp["Message"]}
56         Return From Keyword If  ${status} == ${TRUE}  ${entry}
57    END
58
59    Fail  No ${message_id} logging entry found.
60
61
62Get Error Logs
63    [Documentation]  Return the BMC error logs as a dictionary.
64    [Arguments]   ${quiet}=1  &{filter_struct_args}
65
66    # Example of call using pre-defined filter args (defined above).
67
68    # ${err_logs}=  Get Error Logs  &{filter_low_severity_errlogs}
69
70    # In this example, all error logs with "Severity" fields that are neither Informational, Debug nor
71    # Notice will be returned.
72
73    # Description of argument(s):
74    # quiet                         Indicates whether this keyword should run without any output to the
75    #                               console, 0 = verbose, 1 = quiet.
76    # filter_struct_args            filter_struct args (e.g. filter_dict, regex, etc.) to be passed directly
77    #                               to the Filter Struct keyword.  See its prolog for details.
78
79    #  The length of the returned dictionary indicates how many logs there are.
80
81    # Use 'Print Error Logs' to print.  Example:
82
83    # Print Error Logs  ${error_logs}  Message.
84
85    ${status}  ${error_logs}=  Run Keyword And Ignore Error  Read Properties
86    ...  /xyz/openbmc_project/logging/entry/enumerate  timeout=30  quiet=${quiet}
87    Return From Keyword If  '${status}' == 'FAIL'  &{EMPTY}
88    ${num_filter_struct_args}=  Get Length  ${filter_struct_args}
89    Return From Keyword If  '${num_filter_struct_args}' == '${0}'  ${error_logs}
90    ${filtered_error_logs}=  Filter Struct  ${error_logs}  &{filter_struct_args}
91    [Return]  ${filtered_error_logs}
92
93
94Get IPMI SEL Setting
95    [Documentation]  Returns status for given IPMI SEL setting.
96    [Arguments]  ${setting}
97    # Description of argument(s):
98    # setting  SEL setting which needs to be read(e.g. "Last Add Time").
99
100    ${resp}=  Run IPMI Standard Command  sel info
101
102    ${setting_line}=  Get Lines Containing String  ${resp}  ${setting}
103    ...  case-insensitive
104    ${setting_status}=  Fetch From Right  ${setting_line}  :${SPACE}
105
106    [Return]  ${setting_status}
107
108
109Verify Watchdog Errorlog Content
110    [Documentation]  Verify watchdog errorlog content.
111    # Example:
112    # "/xyz/openbmc_project/logging/entry/1":
113    #  {
114    #      "AdditionalData": [],
115    #      "Id": 1,
116    #      "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut",
117    #      "Resolved": 0,
118    #      "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error",
119    #      "Timestamp": 1492715244828,
120    #      "Associations": []
121    # },
122
123    ${elog_entry}=  Get URL List  ${BMC_LOGGING_ENTRY}
124    ${elog}=  Read Properties  ${elog_entry[0]}
125    Should Be Equal As Strings
126    ...  ${elog["Message"]}  org.open_power.Host.Boot.Error.WatchdogTimedOut
127    ...  msg=Watchdog timeout error log was not found.
128    Should Be Equal As Strings
129    ...  ${elog["Severity"]}  xyz.openbmc_project.Logging.Entry.Level.Error
130    ...  msg=Watchdog timeout severity unexpected value.
131
132
133Logging Test Binary Exist
134    [Documentation]  Verify existence of prerequisite logging-test.
135    Open Connection And Log In
136    ${out}  ${stderr}=  Execute Command
137    ...  which /tmp/tarball/bin/logging-test  return_stderr=True
138    Should Be Empty  ${stderr}  msg=Logging Test stderr is non-empty.
139    Should Contain  ${out}  logging-test
140    ...  msg=Logging test returned unexpected result.
141
142Clear Existing Error Logs
143    [Documentation]  If error log isn't empty, reboot the BMC to clear the log.
144    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}${1}
145    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
146    Initiate BMC Reboot
147    Wait Until Keyword Succeeds  10 min  10 sec
148    ...  Is BMC Ready
149    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}${1}
150    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
151    ...  msg=Could not clear BMC error logs.
152
153
154Create Test PEL Log
155    [Documentation]  Generate test PEL log.
156    [Arguments]  ${pel_type}=Unrecoverable Error
157
158    # Description of argument(s):
159    # pel_type      The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout).
160
161    # Test PEL log entry example:
162    # {
163    #    "0x5000002D": {
164    #            "SRC": "BD8D1002",
165    #            "Message": "An application had an internal failure",
166    #            "PLID": "0x5000002D",
167    #            "CreatorID": "BMC",
168    #            "Subsystem": "BMC Firmware",
169    #            "Commit Time": "02/25/2020  04:47:09",
170    #            "Sev": "Unrecoverable Error",
171    #            "CompID": "0x1000"
172    #    }
173    # }
174
175    Run Keyword If  '${pel_type}' == 'Internal Failure'
176    ...   BMC Execute Command  ${CMD_INTERNAL_FAILURE}
177    ...  ELSE IF  '${pel_type}' == 'FRU Callout'
178    ...   BMC Execute Command  ${CMD_FRU_CALLOUT}
179    ...  ELSE IF  '${pel_type}' == 'Procedure And Symbolic FRU Callout'
180    ...   BMC Execute Command  ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT}
181    ...  ELSE IF  '${pel_type}' == 'Unrecoverable Error'
182    ...   BMC Execute Command  ${CMD_UNRECOVERABLE_ERROR}
183
184
185Create Test Error Log
186    [Documentation]  Generate test error log.
187    # Test error log entry example:
188    # "/xyz/openbmc_project/logging/entry/1":  {
189    #     "AdditionalData": [
190    #         "STRING=FOO"
191    #     ],
192    #     "Id": 1,
193    #     "Message": "example.xyz.openbmc_project.Example.Elog.AutoTestSimple",
194    #     "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error",
195    #     "Timestamp": 1487743963328,
196    #     "Associations": []
197    # }
198    BMC Execute Command  /tmp/tarball/bin/logging-test -c AutoTestSimple
199
200Count Error Entries
201    [Documentation]  Count Error entries.
202    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}
203    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
204    ...  msg=Failed to get error logs.
205    ${jsondata}=  To JSON  ${resp.content}
206    ${count}=  Get Length  ${jsondata["data"]}
207    [Return]  ${count}
208
209Verify Test Error Log
210    [Documentation]  Verify test error log entries.
211    ${elog_entry}=  Get URL List  ${BMC_LOGGING_ENTRY}
212    ${entry_id}=  Read Attribute  ${elog_entry[0]}  Message
213    Should Be Equal  ${entry_id}
214    ...  example.xyz.openbmc_project.Example.Elog.AutoTestSimple
215    ...  msg=Error log not from AutoTestSimple.
216    ${entry_id}=  Read Attribute  ${elog_entry[0]}  Severity
217    Should Be Equal  ${entry_id}
218    ...  xyz.openbmc_project.Logging.Entry.Level.Error
219    ...  msg=Error log severity mismatch.
220
221Delete Error Logs And Verify
222    [Documentation]  Delete all error logs and verify.
223    Delete All Error Logs
224    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}list  quiet=${1}
225    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
226    ...  msg=Error logs not deleted as expected.
227
228
229Install Tarball
230    [Documentation]  Install tarball on BMC.
231    Should Not Be Empty  ${DEBUG_TARBALL_PATH}
232    ...  msg=Debug tarball path value is required.
233    BMC Execute Command  rm -rf /tmp/tarball
234    Install Debug Tarball On BMC  ${DEBUG_TARBALL_PATH}
235
236
237Get Event Logs
238    [Documentation]  Get all available EventLog entries.
239
240    #{
241    #  "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection",
242    #  "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries",
243    #  "@odata.type": "#LogEntryCollection.LogEntryCollection",
244    #  "Description": "Collection of System Event Log Entries",
245    #  "Members": [
246    #  {
247    #    "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry",
248    #    "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1",
249    #    "@odata.type": "#LogEntry.v1_4_0.LogEntry",
250    #    "Created": "2019-05-29T13:19:27+00:00",
251    #    "EntryType": "Event",
252    #    "Id": "1",
253    #    "Message": "org.open_power.Host.Error.Event",
254    #    "Name": "System DBus Event Log Entry",
255    #    "Severity": "Critical"
256    #  }
257    #  ],
258    #  "Members@odata.count": 1,
259    #  "Name": "System Event Log Entries"
260    #}
261
262    ${members}=  Redfish.Get Attribute  ${EVENT_LOG_URI}Entries  Members
263    [Return]  ${members}
264
265
266Get Redfish Event Logs
267    [Documentation]  Pack the list of all available EventLog entries in dictionary.
268    [Arguments]   ${quiet}=1  &{filter_struct_args}
269
270    # Description of argument(s):
271    # quiet                  Indicates whether this keyword should run without any output to the
272    #                        console, 0 = verbose, 1 = quiet.
273    # filter_struct_args     filter_struct args (e.g. filter_dict, regex, etc.) to be passed
274    #                        directly to the Filter Struct keyword.  See its prolog for details.
275
276    ${packed_dict}=  Create Dictionary
277    ${error_logs}=  Get Event Logs
278
279    FOR  ${idx}   IN  @{error_logs}
280       Set To Dictionary  ${packed_dict}    ${idx['@odata.id']}=${idx}
281    END
282
283    ${num_filter_struct_args}=  Get Length  ${filter_struct_args}
284    Return From Keyword If  '${num_filter_struct_args}' == '${0}'  &{packed_dict}
285    ${filtered_error_logs}=  Filter Struct  ${packed_dict}  &{filter_struct_args}
286
287    [Return]  ${filtered_error_logs}
288
289
290Get Event Logs Not Ok
291    [Documentation]  Get all event logs where the 'Severity' is not 'OK'.
292
293    ${members}=  Get Event Logs
294    ${severe_logs}=  Evaluate  [elog for elog in $members if elog['Severity'] != 'OK']
295    [Return]  ${severe_logs}
296
297
298Get Number Of Event Logs
299    [Documentation]  Return the number of EventLog members.
300
301    ${members}=  Get Event Logs
302    ${num_members}=  Get Length  ${members}
303    [Return]  ${num_members}
304
305
306Redfish Purge Event Log
307    [Documentation]  Do Redfish EventLog purge.
308
309    ${target_action}=  redfish_utils.Get Target Actions
310    ...  /redfish/v1/Systems/system/LogServices/EventLog/  LogService.ClearLog
311    Redfish.Post  ${target_action}  body={'target': '${target_action}'}
312    ...  valid_status_codes=[${HTTP_OK}, ${HTTP_NO_CONTENT}]
313
314
315Event Log Should Not Exist
316    [Documentation]  Event log entries should not exist.
317
318    ${elogs}=  Get Event Logs
319    Should Be Empty  ${elogs}  msg=System event log entry is not empty.
320