1*** Settings ***
2Documentation    Error logging utility keywords.
3
4Resource        rest_client.robot
5Resource        bmc_redfish_utils.robot
6Variables       ../data/variables.py
7Variables       ../data/pel_variables.py
8
9*** Variables ***
10
11
12# Define variables for use by callers of 'Get Error Logs'.
13# Old regex: ${low_severity_errlog_regex}  \\.(Informational|Notice|Debug|OK)$
14# Simplified regex string search
15${low_severity_errlog_regex}   Informational|Notice|Debug|OK
16&{low_severity_errlog_filter}  Severity=${low_severity_errlog_regex}
17&{low_severity_errlog_filter_args}  filter_dict=${low_severity_errlog_filter}  regex=${True}  invert=${True}
18# The following is equivalent to &{low_severity_errlog_filter_args} but the name may be more intuitive for
19# users. Example usage:
20# ${err_logs}=  Get Error Logs  &{filter_low_severity_errlogs}
21&{filter_low_severity_errlogs}  &{low_severity_errlog_filter_args}
22
23*** Keywords ***
24
25Filter Expected Logging Events
26    [Documentation]  Get redfish logging entry, remove the user input expected
27    ...              log event and return the object list.
28    [Arguments]  ${expected_event}=None
29
30    # Description of argument(s):
31    # expected_eventd   Event log list.
32
33    ${all_event_list}=  Get Redfish Event Logs
34    Remove Values From List  ${all_event_list}  ${expected_event}
35
36    RETURN  ${all_event_list}
37
38
39Get Logging Entry List
40    [Documentation]  Get logging entry and return the object list.
41
42    ${entry_list}=  Create List
43    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}list  quiet=${1}
44    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
45
46    FOR  ${entry}  IN  @{resp.json()["data"]}
47        Continue For Loop If  '${entry.rsplit('/', 1)[1]}' == 'callout'
48        Append To List  ${entry_list}  ${entry}
49    END
50
51    # Logging entries list.
52    # ['/xyz/openbmc_project/logging/entry/14',
53    #  '/xyz/openbmc_project/logging/entry/15']
54    RETURN  ${entry_list}
55
56
57Logging Entry Should Exist
58    [Documentation]  Find the matching message id and return the entry id.
59    [Arguments]  ${message_id}
60
61    # Description of argument(s):
62    # message_id    Logging message string.
63    #               Example: "xyz.openbmc_project.Common.Error.InternalFailure"
64
65    @{elog_entries}=  Get Logging Entry List
66
67    FOR  ${entry}  IN  @{elog_entries}
68         ${resp}=  Read Properties  ${entry}
69         ${status}=  Run Keyword And Return Status
70         ...  Should Be Equal As Strings  ${message_id}  ${resp["Message"]}
71         Return From Keyword If  ${status} == ${TRUE}  ${entry}
72    END
73
74    Fail  No ${message_id} logging entry found.
75
76
77Get Error Logs
78    [Documentation]  Return the BMC error logs as a dictionary.
79    [Arguments]   ${quiet}=1  &{filter_struct_args}
80
81    # Example of call using pre-defined filter args (defined above).
82
83    # ${err_logs}=  Get Error Logs  &{filter_low_severity_errlogs}
84
85    # In this example, all error logs with "Severity" fields that are neither Informational, Debug nor
86    # Notice will be returned.
87
88    # Description of argument(s):
89    # quiet                         Indicates whether this keyword should run without any output to the
90    #                               console, 0 = verbose, 1 = quiet.
91    # filter_struct_args            filter_struct args (e.g. filter_dict, regex, etc.) to be passed directly
92    #                               to the Filter Struct keyword.  See its prolog for details.
93
94    #  The length of the returned dictionary indicates how many logs there are.
95
96    # Use 'Print Error Logs' to print.  Example:
97
98    # Print Error Logs  ${error_logs}  Message.
99
100    ${status}  ${error_logs}=  Run Keyword And Ignore Error  Read Properties
101    ...  /xyz/openbmc_project/logging/entry/enumerate  timeout=30  quiet=${quiet}
102    Return From Keyword If  '${status}' == 'FAIL'  &{EMPTY}
103    ${num_filter_struct_args}=  Get Length  ${filter_struct_args}
104    Return From Keyword If  '${num_filter_struct_args}' == '${0}'  ${error_logs}
105    ${filtered_error_logs}=  Filter Struct  ${error_logs}  &{filter_struct_args}
106    RETURN  ${filtered_error_logs}
107
108
109Get IPMI SEL Setting
110    [Documentation]  Returns status for given IPMI SEL setting.
111    [Arguments]  ${setting}
112    # Description of argument(s):
113    # setting  SEL setting which needs to be read(e.g. "Last Add Time").
114
115    ${resp}=  Run IPMI Standard Command  sel info
116
117    ${setting_line}=  Get Lines Containing String  ${resp}  ${setting}
118    ...  case-insensitive
119    ${setting_status}=  Fetch From Right  ${setting_line}  :${SPACE}
120
121    RETURN  ${setting_status}
122
123
124Verify Watchdog Errorlog Content
125    [Documentation]  Verify watchdog errorlog content.
126    # Example:
127    # "/xyz/openbmc_project/logging/entry/1":
128    #  {
129    #      "AdditionalData": [],
130    #      "Id": 1,
131    #      "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut",
132    #      "Resolved": 0,
133    #      "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error",
134    #      "Timestamp": 1492715244828,
135    #      "Associations": []
136    # },
137
138    ${elog_entry}=  Get URL List  ${BMC_LOGGING_ENTRY}
139    ${elog}=  Read Properties  ${elog_entry[0]}
140    Should Be Equal As Strings
141    ...  ${elog["Message"]}  org.open_power.Host.Boot.Error.WatchdogTimedOut
142    ...  msg=Watchdog timeout error log was not found.
143    Should Be Equal As Strings
144    ...  ${elog["Severity"]}  xyz.openbmc_project.Logging.Entry.Level.Error
145    ...  msg=Watchdog timeout severity unexpected value.
146
147
148Logging Test Binary Exist
149    [Documentation]  Verify existence of prerequisite logging-test.
150    ${stdout}  ${stderr}  ${rc}=
151    ...  BMC Execute Command  test -f /tmp/tarball/bin/logging-test  print_out=1
152    Should Be Empty  ${stderr}  msg=Logging Test stderr is non-empty.
153
154
155Clear Existing Error Logs
156    [Documentation]  If error log isn't empty, reboot the BMC to clear the log.
157    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}${1}
158    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
159    Initiate BMC Reboot
160    Wait Until Keyword Succeeds  10 min  10 sec
161    ...  Is BMC Ready
162    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}${1}
163    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
164    ...  msg=Could not clear BMC error logs.
165
166
167Create Test PEL Log
168    [Documentation]  Generate test PEL log.
169    [Arguments]  ${pel_type}=Unrecoverable Error
170
171    # Description of argument(s):
172    # pel_type      The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout).
173
174    # Test PEL log entry example:
175    # {
176    #    "0x5000002D": {
177    #            "SRC": "BD8D1002",
178    #            "Message": "An application had an internal failure",
179    #            "PLID": "0x5000002D",
180    #            "CreatorID": "BMC",
181    #            "Subsystem": "BMC Firmware",
182    #            "Commit Time": "02/25/2020  04:47:09",
183    #            "Sev": "Unrecoverable Error",
184    #            "CompID": "0x1000"
185    #    }
186    # }
187
188    Run Keyword If  '${pel_type}' == 'Internal Failure'
189    ...   BMC Execute Command  ${CMD_INTERNAL_FAILURE}
190    ...  ELSE IF  '${pel_type}' == 'FRU Callout'
191    ...   BMC Execute Command  ${CMD_FRU_CALLOUT}
192    ...  ELSE IF  '${pel_type}' == 'Procedure And Symbolic FRU Callout'
193    ...   BMC Execute Command  ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT}
194    ...  ELSE IF  '${pel_type}' == 'Unrecoverable Error'
195    ...   BMC Execute Command  ${CMD_UNRECOVERABLE_ERROR}
196
197
198Create Test Error Log
199    [Documentation]  Generate test error log.
200    # Test error log entry example:
201    # "/xyz/openbmc_project/logging/entry/1":  {
202    #     "AdditionalData": [
203    #         "STRING=FOO"
204    #     ],
205    #     "Id": 1,
206    #     "Message": "example.xyz.openbmc_project.Example.Elog.AutoTestSimple",
207    #     "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error",
208    #     "Timestamp": 1487743963328,
209    #     "Associations": []
210    # }
211    BMC Execute Command  /tmp/tarball/bin/logging-test -c AutoTestSimple
212
213Count Error Entries
214    [Documentation]  Count Error entries.
215    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}
216    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
217    ...  msg=Failed to get error logs.
218    ${count}=  Get Length  ${resp.json()["data"]}
219    RETURN  ${count}
220
221Verify Test Error Log
222    [Documentation]  Verify test error log entries.
223    ${elog_entry}=  Get URL List  ${BMC_LOGGING_ENTRY}
224    ${entry_id}=  Read Attribute  ${elog_entry[0]}  Message
225    Should Be Equal  ${entry_id}
226    ...  example.xyz.openbmc_project.Example.Elog.AutoTestSimple
227    ...  msg=Error log not from AutoTestSimple.
228    ${entry_id}=  Read Attribute  ${elog_entry[0]}  Severity
229    Should Be Equal  ${entry_id}
230    ...  xyz.openbmc_project.Logging.Entry.Level.Error
231    ...  msg=Error log severity mismatch.
232
233Delete Error Logs And Verify
234    [Documentation]  Delete all error logs and verify.
235    Delete All Error Logs
236    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}list  quiet=${1}
237    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
238    ...  msg=Error logs not deleted as expected.
239
240
241Install Tarball
242    [Documentation]  Install tarball on BMC.
243    Should Not Be Empty  ${DEBUG_TARBALL_PATH}
244    ...  msg=Debug tarball path value is required.
245    BMC Execute Command  rm -rf /tmp/tarball
246    Install Debug Tarball On BMC  ${DEBUG_TARBALL_PATH}
247
248
249Get Event Logs
250    [Documentation]  Get all available EventLog entries.
251
252    #{
253    #  "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection",
254    #  "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries",
255    #  "@odata.type": "#LogEntryCollection.LogEntryCollection",
256    #  "Description": "Collection of System Event Log Entries",
257    #  "Members": [
258    #  {
259    #    "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry",
260    #    "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1",
261    #    "@odata.type": "#LogEntry.v1_4_0.LogEntry",
262    #    "Created": "2019-05-29T13:19:27+00:00",
263    #    "EntryType": "Event",
264    #    "Id": "1",
265    #    "Message": "org.open_power.Host.Error.Event",
266    #    "Name": "System DBus Event Log Entry",
267    #    "Severity": "Critical"
268    #  }
269    #  ],
270    #  "Members@odata.count": 1,
271    #  "Name": "System Event Log Entries"
272    #}
273
274    ${members}=  Redfish.Get Attribute  ${EVENT_LOG_URI}Entries  Members
275    RETURN  ${members}
276
277
278Get Redfish Event Logs
279    [Documentation]  Pack the list of all available EventLog entries in dictionary.
280    [Arguments]   ${quiet}=1  &{filter_struct_args}
281
282    # Description of argument(s):
283    # quiet                  Indicates whether this keyword should run without any output to the
284    #                        console, 0 = verbose, 1 = quiet.
285    # filter_struct_args     filter_struct args (e.g. filter_dict, regex, etc.) to be passed
286    #                        directly to the Filter Struct keyword.  See its prolog for details.
287
288    ${packed_dict}=  Create Dictionary
289    ${error_logs}=  Get Event Logs
290
291    FOR  ${idx}   IN  @{error_logs}
292       Set To Dictionary  ${packed_dict}    ${idx['@odata.id']}=${idx}
293    END
294
295    ${num_filter_struct_args}=  Get Length  ${filter_struct_args}
296    Return From Keyword If  '${num_filter_struct_args}' == '${0}'  &{packed_dict}
297    ${filtered_error_logs}=  Filter Struct  ${packed_dict}  &{filter_struct_args}
298
299    RETURN  ${filtered_error_logs}
300
301
302Get Event Logs Not Ok
303    [Documentation]  Get all event logs where the 'Severity' is not 'OK'.
304
305    ${members}=  Get Event Logs
306    ${severe_logs}=  Evaluate  [elog for elog in $members if elog['Severity'] != 'OK']
307    RETURN  ${severe_logs}
308
309
310Get Number Of Event Logs
311    [Documentation]  Return the number of EventLog members.
312
313    ${members}=  Get Event Logs
314    ${num_members}=  Get Length  ${members}
315    RETURN  ${num_members}
316
317
318Redfish Purge Event Log
319    [Documentation]  Do Redfish EventLog purge.
320
321    ${target_action}=  redfish_utils.Get Target Actions
322    ...  /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/  LogService.ClearLog
323    Redfish.Post  ${target_action}  body={'target': '${target_action}'}
324    ...  valid_status_codes=[${HTTP_OK}, ${HTTP_NO_CONTENT}]
325
326
327Event Log Should Not Exist
328    [Documentation]  Event log entries should not exist.
329
330    ${elogs}=  Get Event Logs
331    Should Be Empty  ${elogs}  msg=System event log entry is not empty.
332
333
334Redfish Clear PostCodes
335    [Documentation]  Do Redfish PostCodes purge from system.
336
337    ${target_action}=  redfish_utils.Get Target Actions
338    ...  /redfish/v1/Systems/${SYSTEM_ID}/LogServices/PostCodes/  LogService.ClearLog
339    Redfish.Post  ${target_action}  body={'target': '${target_action}'}
340    ...  valid_status_codes=[${HTTP_OK}, ${HTTP_NO_CONTENT}]
341
342
343Redfish Get PostCodes
344    [Documentation]  Perform Redfish GET request and return the PostCodes entries as a list of dictionaries.
345
346    # Formatted example output from Rprint vars  members
347    # members:
348    #  [0]:
349    #    [@odata.id]:               /redfish/v1/Systems/system/LogServices/PostCodes/Entries/B1-1
350    #    [@odata.type]:             #LogEntry.v1_8_0.LogEntry
351    #    [AdditionalDataURI]:       /redfish/v1/Systems/system/LogServices/PostCodes/Entries/B1-1/attachment
352    #    [Created]:                 2022-08-06T04:38:10+00:00
353    #    [EntryType]:               Event
354    #    [Id]:                      B1-1
355    #    [Message]:                 Message": "Boot Count: 4: TS Offset: 0.0033; POST Code: 0x43
356    #    [MessageArgs]:
357    #      [0]:                    4
358    #      [1]:                    0.0033
359    #      [2]:                    0x43
360    #    [MessageId]:              OpenBMC.0.2.BIOSPOSTCodeASCII
361    #    [Name]:                   POST Code Log Entry
362    #    [Severity]:               OK
363
364    ${members}=  Redfish.Get Attribute  /redfish/v1/Systems/${SYSTEM_ID}/LogServices/PostCodes/Entries  Members
365    ...  valid_status_codes=[${HTTP_OK}, ${HTTP_NO_CONTENT}]
366
367    RETURN  ${members}
368