1*** Settings ***
2Documentation    Error logging utility keywords.
3
4Resource        rest_client.robot
5Variables       ../data/variables.py
6Variables       ../data/pel_variables.py
7
8*** Variables ***
9
10
11# Define variables for use by callers of 'Get Error Logs'.
12${low_severity_errlog_regex}  \\.(Informational|Notice|Debug)$
13&{low_severity_errlog_filter}  Severity=${low_severity_errlog_regex}
14&{low_severity_errlog_filter_args}  filter_dict=${low_severity_errlog_filter}  regex=${True}  invert=${True}
15# The following is equivalent to &{low_severity_errlog_filter_args} but the name may be more intuitive for
16# users. Example usage:
17# ${err_logs}=  Get Error Logs  &{filter_low_severity_errlogs}
18&{filter_low_severity_errlogs}  &{low_severity_errlog_filter_args}
19
20*** Keywords ***
21
22Get Logging Entry List
23    [Documentation]  Get logging entry and return the object list.
24
25    ${entry_list}=  Create List
26    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}list  quiet=${1}
27    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
28    ${jsondata}=  To JSON  ${resp.content}
29
30    FOR  ${entry}  IN  @{jsondata["data"]}
31        Continue For Loop If  '${entry.rsplit('/', 1)[1]}' == 'callout'
32        Append To List  ${entry_list}  ${entry}
33    END
34
35    # Logging entries list.
36    # ['/xyz/openbmc_project/logging/entry/14',
37    #  '/xyz/openbmc_project/logging/entry/15']
38    [Return]  ${entry_list}
39
40
41Logging Entry Should Exist
42    [Documentation]  Find the matching message id and return the entry id.
43    [Arguments]  ${message_id}
44
45    # Description of argument(s):
46    # message_id    Logging message string.
47    #               Example: "xyz.openbmc_project.Common.Error.InternalFailure"
48
49    @{elog_entries}=  Get Logging Entry List
50
51    FOR  ${entry}  IN  @{elog_entries}
52         ${resp}=  Read Properties  ${entry}
53         ${status}=  Run Keyword And Return Status
54         ...  Should Be Equal As Strings  ${message_id}  ${resp["Message"]}
55         Return From Keyword If  ${status} == ${TRUE}  ${entry}
56    END
57
58    Fail  No ${message_id} logging entry found.
59
60
61Get Error Logs
62    [Documentation]  Return the BMC error logs as a dictionary.
63    [Arguments]   ${quiet}=1  &{filter_struct_args}
64
65    # Example of call using pre-defined filter args (defined above).
66
67    # ${err_logs}=  Get Error Logs  &{filter_low_severity_errlogs}
68
69    # In this example, all error logs with "Severity" fields that are neither Informational, Debug nor
70    # Notice will be returned.
71
72    # Description of argument(s):
73    # quiet                         Indicates whether this keyword should run without any output to the
74    #                               console, 0 = verbose, 1 = quiet.
75    # filter_struct_args            filter_struct args (e.g. filter_dict, regex, etc.) to be passed directly
76    #                               to the Filter Struct keyword.  See its prolog for details.
77
78    #  The length of the returned dictionary indicates how many logs there are.
79
80    # Use 'Print Error Logs' to print.  Example:
81
82    # Print Error Logs  ${error_logs}  Message.
83
84    ${status}  ${error_logs}=  Run Keyword And Ignore Error  Read Properties
85    ...  /xyz/openbmc_project/logging/entry/enumerate  timeout=30  quiet=${quiet}
86    Return From Keyword If  '${status}' == 'FAIL'  &{EMPTY}
87    ${num_filter_struct_args}=  Get Length  ${filter_struct_args}
88    Return From Keyword If  '${num_filter_struct_args}' == '${0}'  ${error_logs}
89    ${filtered_error_logs}=  Filter Struct  ${error_logs}  &{filter_struct_args}
90    [Return]  ${filtered_error_logs}
91
92
93Get IPMI SEL Setting
94    [Documentation]  Returns status for given IPMI SEL setting.
95    [Arguments]  ${setting}
96    # Description of argument(s):
97    # setting  SEL setting which needs to be read(e.g. "Last Add Time").
98
99    ${resp}=  Run IPMI Standard Command  sel info
100
101    ${setting_line}=  Get Lines Containing String  ${resp}  ${setting}
102    ...  case-insensitive
103    ${setting_status}=  Fetch From Right  ${setting_line}  :${SPACE}
104
105    [Return]  ${setting_status}
106
107
108Verify Watchdog Errorlog Content
109    [Documentation]  Verify watchdog errorlog content.
110    # Example:
111    # "/xyz/openbmc_project/logging/entry/1":
112    #  {
113    #      "AdditionalData": [],
114    #      "Id": 1,
115    #      "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut",
116    #      "Resolved": 0,
117    #      "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error",
118    #      "Timestamp": 1492715244828,
119    #      "Associations": []
120    # },
121
122    ${elog_entry}=  Get URL List  ${BMC_LOGGING_ENTRY}
123    ${elog}=  Read Properties  ${elog_entry[0]}
124    Should Be Equal As Strings
125    ...  ${elog["Message"]}  org.open_power.Host.Boot.Error.WatchdogTimedOut
126    ...  msg=Watchdog timeout error log was not found.
127    Should Be Equal As Strings
128    ...  ${elog["Severity"]}  xyz.openbmc_project.Logging.Entry.Level.Error
129    ...  msg=Watchdog timeout severity unexpected value.
130
131
132Logging Test Binary Exist
133    [Documentation]  Verify existence of prerequisite logging-test.
134    Open Connection And Log In
135    ${out}  ${stderr}=  Execute Command
136    ...  which /tmp/tarball/bin/logging-test  return_stderr=True
137    Should Be Empty  ${stderr}  msg=Logging Test stderr is non-empty.
138    Should Contain  ${out}  logging-test
139    ...  msg=Logging test returned unexpected result.
140
141Clear Existing Error Logs
142    [Documentation]  If error log isn't empty, reboot the BMC to clear the log.
143    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}${1}
144    Return From Keyword If  ${resp.status_code} == ${HTTP_NOT_FOUND}
145    Initiate BMC Reboot
146    Wait Until Keyword Succeeds  10 min  10 sec
147    ...  Is BMC Ready
148    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}${1}
149    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
150    ...  msg=Could not clear BMC error logs.
151
152
153Create Test PEL Log
154    [Documentation]  Generate test PEL log.
155    [Arguments]  ${pel_type}=Internal Failure
156
157    # Description of argument(s):
158    # pel_type      The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout).
159
160    # Test PEL log entry example:
161    # {
162    #    "0x5000002D": {
163    #            "SRC": "BD8D1002",
164    #            "Message": "An application had an internal failure",
165    #            "PLID": "0x5000002D",
166    #            "CreatorID": "BMC",
167    #            "Subsystem": "BMC Firmware",
168    #            "Commit Time": "02/25/2020  04:47:09",
169    #            "Sev": "Unrecoverable Error",
170    #            "CompID": "0x1000"
171    #    }
172    # }
173
174    Run Keyword If  '${pel_type}' == 'Internal Failure'
175    ...   BMC Execute Command  ${CMD_INTERNAL_FAILURE}
176    ...  ELSE IF  '${pel_type}' == 'FRU Callout'
177    ...   BMC Execute Command  ${CMD_FRU_CALLOUT}
178    ...  ELSE IF  '${pel_type}' == 'Procedure And Symbolic FRU Callout'
179    ...   BMC Execute Command  ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT}
180
181
182Create Test Error Log
183    [Documentation]  Generate test error log.
184    # Test error log entry example:
185    # "/xyz/openbmc_project/logging/entry/1":  {
186    #     "AdditionalData": [
187    #         "STRING=FOO"
188    #     ],
189    #     "Id": 1,
190    #     "Message": "example.xyz.openbmc_project.Example.Elog.AutoTestSimple",
191    #     "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error",
192    #     "Timestamp": 1487743963328,
193    #     "Associations": []
194    # }
195    BMC Execute Command  /tmp/tarball/bin/logging-test -c AutoTestSimple
196
197Count Error Entries
198    [Documentation]  Count Error entries.
199    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}
200    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
201    ...  msg=Failed to get error logs.
202    ${jsondata}=  To JSON  ${resp.content}
203    ${count}=  Get Length  ${jsondata["data"]}
204    [Return]  ${count}
205
206Verify Test Error Log
207    [Documentation]  Verify test error log entries.
208    ${elog_entry}=  Get URL List  ${BMC_LOGGING_ENTRY}
209    ${entry_id}=  Read Attribute  ${elog_entry[0]}  Message
210    Should Be Equal  ${entry_id}
211    ...  example.xyz.openbmc_project.Example.Elog.AutoTestSimple
212    ...  msg=Error log not from AutoTestSimple.
213    ${entry_id}=  Read Attribute  ${elog_entry[0]}  Severity
214    Should Be Equal  ${entry_id}
215    ...  xyz.openbmc_project.Logging.Entry.Level.Error
216    ...  msg=Error log severity mismatch.
217
218Delete Error Logs And Verify
219    [Documentation]  Delete all error logs and verify.
220    Delete All Error Logs
221    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}list  quiet=${1}
222    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_NOT_FOUND}
223    ...  msg=Error logs not deleted as expected.
224
225
226Install Tarball
227    [Documentation]  Install tarball on BMC.
228    Should Not Be Empty  ${DEBUG_TARBALL_PATH}
229    ...  msg=Debug tarball path value is required.
230    BMC Execute Command  rm -rf /tmp/tarball
231    Install Debug Tarball On BMC  ${DEBUG_TARBALL_PATH}
232
233
234Get Event Logs
235    [Documentation]  Get all available EventLog entries.
236
237    #{
238    #  "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection",
239    #  "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries",
240    #  "@odata.type": "#LogEntryCollection.LogEntryCollection",
241    #  "Description": "Collection of System Event Log Entries",
242    #  "Members": [
243    #  {
244    #    "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry",
245    #    "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1",
246    #    "@odata.type": "#LogEntry.v1_4_0.LogEntry",
247    #    "Created": "2019-05-29T13:19:27+00:00",
248    #    "EntryType": "Event",
249    #    "Id": "1",
250    #    "Message": "org.open_power.Host.Error.Event",
251    #    "Name": "System DBus Event Log Entry",
252    #    "Severity": "Critical"
253    #  }
254    #  ],
255    #  "Members@odata.count": 1,
256    #  "Name": "System Event Log Entries"
257    #}
258
259    ${members}=  Redfish.Get Attribute  ${EVENT_LOG_URI}Entries  Members
260    [Return]  ${members}
261
262
263Get Event Logs Not Ok
264    [Documentation]  Get all event logs where the 'Severity' is not 'OK'.
265
266    ${members}=  Get Event Logs
267    ${severe_logs}=  Evaluate  [elog for elog in $members if elog['Severity'] != 'OK']
268    [Return]  ${severe_logs}
269
270
271Get Number Of Event Logs
272    [Documentation]  Return the number of EventLog members.
273
274    ${members}=  Get Event Logs
275    ${num_members}=  Get Length  ${members}
276    [Return]  ${num_members}
277
278
279Redfish Purge Event Log
280    [Documentation]  Do Redfish EventLog purge.
281
282    ${target_action}=  redfish_utils.Get Target Actions
283    ...  /redfish/v1/Systems/system/LogServices/EventLog/  LogService.ClearLog
284    Redfish.Post  ${target_action}  body={'target': '${target_action}'}
285    ...  valid_status_codes=[${HTTP_OK}, ${HTTP_NO_CONTENT}]
286
287