1*** Settings *** 2Documentation Inventory of hardware resources under systems. 3 4Resource ../../../lib/bmc_redfish_resource.robot 5Resource ../../../lib/bmc_redfish_utils.robot 6Resource ../../../lib/logging_utils.robot 7Resource ../../../lib/openbmc_ffdc.robot 8Resource ../../../lib/ipmi_client.robot 9Library ../../../lib/logging_utils.py 10 11Suite Setup Suite Setup Execution 12Suite Teardown Suite Teardown Execution 13Test Setup Test Setup Execution 14Test Teardown Test Teardown Execution 15 16Test Tags Event_Logging 17 18** Variables *** 19 20${max_num_event_logs} ${200} 21 22*** Test Cases *** 23 24Event Log Check After BMC Reboot 25 [Documentation] Check event log after BMC rebooted. 26 [Tags] Event_Log_Check_After_BMC_Reboot 27 28 Redfish Purge Event Log 29 Event Log Should Not Exist 30 31 Redfish OBMC Reboot (off) 32 33 Redfish.Login 34 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 35 36 Event Log Should Not Exist 37 38 39Event Log Check After Host Poweron 40 [Documentation] Check event log after host has booted. 41 [Tags] Event_Log_Check_After_Host_Poweron 42 43 Redfish Purge Event Log 44 Event Log Should Not Exist 45 46 Redfish Power On 47 48 Redfish.Login 49 Event Log Should Not Exist 50 51 52Create Test Event Log And Verify 53 [Documentation] Create event logs and verify via redfish. 54 [Tags] Create_Test_Event_Log_And_Verify 55 56 Create Test Error Log 57 Event Log Should Exist 58 59 60Delete Redfish Event Log And Verify 61 [Documentation] Delete Redfish event log and verify via Redfish. 62 [Tags] Delete_Redfish_Event_Log_And_Verify 63 64 Redfish.Login 65 Redfish Purge Event Log 66 Create Test PEL Log 67 ${elog_entry}= Get Event Logs 68 69 Redfish.Delete /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries/${elog_entry[0]["Id"]} 70 71 ${error_entries}= Get Redfish Error Entries 72 Should Be Empty ${error_entries} 73 74 75Test Event Log Persistency On Restart 76 [Documentation] Restart logging service and verify event logs. 77 [Tags] Test_Event_Log_Persistency_On_Restart 78 79 Create Test Error Log 80 Event Log Should Exist 81 82 BMC Execute Command 83 ... systemctl restart xyz.openbmc_project.Logging.service 84 Sleep 10s reason=Wait for logging service to restart properly. 85 86 Event Log Should Exist 87 88 89Test Event Entry Numbering Reset On Restart 90 [Documentation] Restart logging service and verify event logs entry starts 91 ... from entry "Id" 1. 92 [Tags] Test_Event_Entry_Numbering_Reset_On_Restart 93 [Setup] Redfish Power Off stack_mode=skip 94 95 #{ 96 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 97 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 98 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 99 # "Description": "Collection of System Event Log Entries", 100 # "Members": [ 101 # { 102 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 103 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 104 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 105 # "Created": "2019-05-29T13:19:27+00:00", 106 # "EntryType": "Event", 107 # "Id": "1", <----- Event log ID 108 # "Message": "org.open_power.Host.Error.Event", 109 # "Name": "System DBus Event Log Entry", 110 # "Severity": "Critical" 111 # } 112 # ], 113 # "Members@odata.count": 1, 114 # "Name": "System Event Log Entries" 115 #} 116 117 Create Test PEL Log 118 Create Test PEL Log 119 Event Log Should Exist 120 121 Redfish Purge Event Log 122 Event Log Should Not Exist 123 124 BMC Execute Command 125 ... systemctl restart xyz.openbmc_project.Logging.service 126 Sleep 10s reason=Wait for logging service to restart properly. 127 128 Create Test PEL Log 129 ${elogs}= Get Event Logs 130 131 # After issuing Redfish purge event log, there will be one informational error log 132 # in BMC with ID 1. Due to this the newly generated error log would have ID as 2. 133 Should Be Equal ${elogs[0]["Id"]} 2 msg=Event log entry is not 2 134 135 136Test Event Log Persistency On Reboot 137 [Documentation] Reboot BMC and verify event log. 138 [Tags] Test_Event_Log_Persistency_On_Reboot 139 140 Redfish Purge Event Log 141 Create Test Error Log 142 Event Log Should Exist 143 144 Redfish OBMC Reboot (off) 145 146 Redfish.Login 147 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 148 149 Event Log Should Exist 150 151 152Create Test Event Log And Verify Time Stamp 153 [Documentation] Create event logs and verify time stamp. 154 [Tags] Create_Test_Event_Log_And_Verify_Time_Stamp 155 156 #{ 157 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 158 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 159 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 160 # "Description": "Collection of System Event Log Entries", 161 # "Members": [ 162 # { 163 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 164 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 165 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 166 # "Created": "2023-05-10T10:26:02.186+00:00", <--- Time stamp 167 # "EntryType": "Event", 168 # "Id": "1", 169 # "Message": "org.open_power.Host.Error.Event", 170 # "Name": "System DBus Event Log Entry", 171 # "Severity": "Critical" 172 # } 173 # ], 174 # "Members@odata.count": 1, 175 # "Name": "System Event Log Entries" 176 #} 177 178 Redfish Purge Event Log 179 180 Create Test Error Log 181 Sleep 2s 182 Create Test Error Log 183 184 ${elog_entry}= Get Event Logs 185 186 # The event log generated is associated with the epoc time and unique 187 # for every error and in increasing time stamp. 188 ${time_stamp1}= Convert Date ${elog_entry[0]["Created"].split('.')[0]} epoch 189 ${time_stamp2}= Convert Date ${elog_entry[1]["Created"].split('.')[0]} epoch 190 191 Should Be True ${time_stamp2} > ${time_stamp1} 192 193 194Verify Setting Error Log As Resolved 195 [Documentation] Verify modified field of error log is updated when error log is marked resolved. 196 [Tags] Verify_Setting_Error_Log_As_Resolved 197 198 Create Test PEL Log 199 ${elog_entry}= Get Event Logs 200 201 # Wait for 5 seconds after creating error log. 202 Sleep 5s 203 204 # Mark error log as resolved by setting it to true. 205 Redfish.Patch ${EVENT_LOG_URI}Entries/${elog_entry[0]["Id"]} body={'Resolved':True} 206 207 ${elog_entry}= Get Event Logs 208 209 # Example error log with resolve field set to true: 210 # { 211 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/2045", 212 # "@odata.type": "#LogEntry.v1_8_0.LogEntry", 213 # "AdditionalDataURI": "/redfish/v1/Systems/system/LogServices/EventLog/attachment/2045", 214 # "Created": "2023-05-10T10:26:02.186+00:00", 215 # "EntryType": "Event", 216 # "Id": "2045", 217 # "Message": "xyz.openbmc_project.Host.Error.Event", 218 # "Modified": "2023-05-10T10:26:02.186+00:00", 219 # "Name": "System Event Log Entry", 220 # "Resolved": true, 221 # "Severity": "OK" 222 # } 223 224 Should Be Equal As Strings ${elog_entry[0]["Resolved"]} True 225 226 # Difference created and modified time of error log should be around 5 seconds. 227 ${creation_time}= Convert Date ${elog_entry[0]["Created"].split('.')[0]} epoch 228 ${modification_time}= Convert Date ${elog_entry[0]["Modified"].split('.')[0]} epoch 229 230 ${diff}= Subtract Date From Date ${modification_time} ${creation_time} 231 ${diff}= Convert To Number ${diff} 232 Should Be True 4 < ${diff} < 8 233 234 235Verify IPMI SEL Delete 236 [Documentation] Verify IPMI SEL delete operation. 237 [Tags] Verify_IPMI_SEL_Delete 238 239 Redfish Purge Event Log 240 Create Test Error Log 241 242 ${sel_list}= Run IPMI Standard Command sel list 243 Should Not Be Equal As Strings ${sel_list} SEL has no entries 244 245 # Example of SEL List: 246 # 4 | 04/21/2017 | 10:51:16 | System Event #0x01 | Undetermined system hardware failure | Asserted 247 248 ${sel_entry}= Fetch from Left ${sel_list} | 249 ${sel_entry}= Evaluate $sel_entry.replace(' ','') 250 ${sel_entry}= Convert To Integer 0x${sel_entry} 251 252 ${sel_delete}= Run IPMI Standard Command sel delete ${sel_entry} 253 Should Be Equal As Strings ${sel_delete} Deleted entry ${sel_entry} 254 ... case_insensitive=True 255 256 ${sel_list}= Run IPMI Standard Command sel list 257 Should Be Equal As Strings ${sel_list} SEL has no entries 258 ... case_insensitive=True 259 260 261Create Test Event Log And Delete 262 [Documentation] Create an event log and delete it. 263 [Tags] Create_Test_Event_Log_And_Delete 264 265 Create Test Error Log 266 Redfish Purge Event Log 267 Event Log Should Not Exist 268 269 270Create Multiple Test Event Logs And Delete All 271 [Documentation] Create multiple event logs and delete all. 272 [Tags] Create_Multiple_Test_Event_Logs_And_Delete_All 273 274 Create Test Error Log 275 Create Test Error Log 276 Create Test Error Log 277 Redfish Purge Event Log 278 Event Log Should Not Exist 279 280 281Create Two Test Event Logs And Delete One 282 [Documentation] Create two event logs and delete the first entry. 283 [Tags] Create_Two_Test_Event_Logs_And_Delete_One 284 [Setup] Redfish Power Off stack_mode=skip 285 286 Redfish Purge Event Log 287 Create Test PEL Log 288 Create Test PEL Log 289 ${error_entries_before}= Get Redfish Error Entries 290 Redfish.Delete /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries/${error_entries_before[0]} 291 292 ${error_entries_after}= Get Redfish Error Entries 293 Should Not Contain ${error_entries_after} ${error_entries_before[0]} 294 Should Contain ${error_entries_after} ${error_entries_before[1]} 295 296 297Verify Watchdog Timedout Event 298 [Documentation] Trigger watchdog timed out and verify event log generated. 299 [Tags] Verify_Watchdog_Timedout_Event 300 [Teardown] Run Keywords Test Teardown Execution AND Redfish Power Off stack_mode=skip 301 302 Redfish Power Off stack_mode=skip 303 304 # Clear errors if there are any. 305 Redfish.Login 306 Redfish Purge Event Log 307 308 # Reference: [Old legacy REST code] Trigger Host Watchdog Error 309 # Currently, no known redfish interface to set to trigger watchdog timer. 310 311 Redfish Initiate Auto Reboot 1000 312 313 # Logging takes time to generate the timeout error. 314 Wait Until Keyword Succeeds 3 min 20 sec Verify Watchdog EventLog Content 315 316 317Verify Event Logs Capping 318 [Documentation] Verify event logs capping. 319 [Tags] Verify_Event_Logs_Capping 320 321 Redfish Purge Event Log 322 323 ${cmd}= Catenate uptime; for i in {1..201}; do /tmp/tarball/bin/logging-test -c 324 ... AutoTestSimple;sleep 1;done; uptime 325 BMC Execute Command ${cmd} 326 327 ${elogs}= Get Event Logs 328 ${count}= Get Length ${elogs} 329 Run Keyword If ${count} > 200 330 ... Fail Error logs created exceeded max capacity 200. 331 332 333Test Event Log Wrapping 334 [Documentation] Verify event log entries wraps when 200 max cap is reached. 335 [Tags] Test_Event_Log_Wrapping 336 337 # Restarting logging service in order to clear logs and get the next log 338 # ID set to 1. 339 BMC Execute Command 340 ... systemctl restart xyz.openbmc_project.Logging.service 341 Sleep 10s reason=Wait for logging service to restart properly. 342 343 # Create ${max_num_event_logs} event logs. 344 ${cmd}= Catenate uptime; for i in {1..${max_num_event_logs}}; do /tmp/tarball/bin/logging-test -c 345 ... AutoTestSimple;sleep 1;done; uptime 346 BMC Execute Command ${cmd} 347 348 # Verify that event logs with IDs 1 and ${max_num_event_logs} exist. 349 ${event_log}= Get Event Logs 350 351 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 352 Rprint Vars log_entries 353 Should Be Equal As Strings ${log_entries[0]["Id"]} 1 354 355 ${log_entries}= Filter Struct ${event_log} [('Id', '${max_num_event_logs}')] 356 Rprint Vars log_entries 357 Should Be Equal As Strings ${log_entries[0]["Id"]} ${max_num_event_logs} 358 359 # Create event log and verify the entry ID, ${max_num_event_logs + 1}. 360 ${next_event_log_id}= Set Variable ${max_num_event_logs + 1} 361 362 Create Test Error Log 363 364 ${event_log}= Get Event Logs 365 366 ${log_entries}= Filter Struct ${event_log} [('Id', '${next_event_log_id}')] 367 Rprint Vars log_entries 368 Should Be Equal As Strings ${log_entries[0]["Id"]} ${next_event_log_id} 369 370 # Event log 1 should be wrapped. 371 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 372 Rprint Vars log_entries 373 374 ${length_log_entries} Get Length ${log_entries} 375 Should Be Equal As Integers ${length_log_entries} 0 376 ... msg=The event log should have wrapped such that entry ID 1 is now purged. 377 378 379Verify Default Value Of Resolved Field Is False For An Error Log Via Redfish 380 [Documentation] Verify the Resolve field status is false for an error log from Redfish. 381 [Tags] Verify_Default_Value_Of_Resolved_Field_Is_False_For_An_Error_Log_Via_Redfish 382 383 Redfish Purge Event Log 384 Create Test Error Log 385 386 # Check resolve field value of created error log. 387 ${elog_entry}= Get Event Logs 388 Should Be Equal ${elog_entry[0]["Resolved"]} ${False} 389 390 391*** Keywords *** 392 393Suite Setup Execution 394 [Documentation] Do test case setup tasks. 395 396 Redfish.Login 397 398 Redfish Purge Event Log 399 400 ${status}= Run Keyword And Return Status Logging Test Binary Exist 401 Run Keyword If ${status} == ${False} Install Tarball 402 403 404Suite Teardown Execution 405 [Documentation] Do the post suite teardown. 406 407 Redfish.Logout 408 409 410Test Setup Execution 411 [Documentation] Do test case setup tasks. 412 413 Redfish Purge Event Log 414 415 ${status}= Run Keyword And Return Status Logging Test Binary Exist 416 Run Keyword If ${status} == ${False} Install Tarball 417 418 419Test Teardown Execution 420 [Documentation] Do the post test teardown. 421 422 FFDC On Test Case Fail 423 Redfish.Login 424 Redfish Purge Event Log 425 426 427Get Redfish Error Entries 428 [Documentation] Return Redfish error ids list. 429 ${error_uris}= redfish_utils.get_member_list /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries 430 ${error_ids}= Create List 431 432 FOR ${error_uri} IN @{error_uris} 433 ${error_id}= Fetch From Right ${error_uri} / 434 Append To List ${error_ids} ${error_id} 435 END 436 437 RETURN ${error_ids} 438 439 440Event Log Should Not Exist 441 [Documentation] Event log entries should not exist. 442 443 ${elogs}= Get Event Logs 444 Should Be Empty ${elogs} msg=System event log entry is not empty. 445 446 447Event Log Should Exist 448 [Documentation] Event log entries should exist. 449 450 ${elogs}= Get Event Logs 451 Should Not Be Empty ${elogs} msg=System event log entry is not empty. 452 453 454Verify Watchdog EventLog Content 455 [Documentation] Verify watchdog event log content. 456 457 # Example: 458 # { 459 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 460 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/31", 461 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 462 # "Created": "2019-05-31T18:41:33+00:00", 463 # "EntryType": "Event", 464 # "Id": "31", 465 # "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut", 466 # "Name": "System DBus Event Log Entry", 467 # "Severity": "Critical" 468 # } 469 470 ${elog_list}= Get Event Logs 471 472 Rprint Vars elog_list 473 474 FOR ${entry} IN @{elog_list} 475 ${found_match}= Run Keyword And Return Status Is Watchdog Error Found ${entry} 476 Exit For Loop If '${found_match}' == 'True' 477 END 478 479 Run Keyword If '${found_match}' == 'False' Fail msg=No watchdog error logged. 480 481 482Is Watchdog Error Found 483 [Documentation] Check if the give log entry matches specific watchdog error. 484 [Arguments] ${elog} 485 486 # Description of argument(s): 487 # elog Error log entry dictionary data. 488 489 Should Contain Any 490 ... ${elog["Message"]} org.open_power.Host.Boot.Error.WatchdogTimedOut 491 ... CEC Hardware - Hostboot-Service Processor Interface 492 ... msg=Watchdog timeout event log was not found. 493 494 Log To Console Matched Found: ${elog} 495