1*** Settings *** 2Documentation Inventory of hardware resources under systems. 3 4Resource ../../../lib/bmc_redfish_resource.robot 5Resource ../../../lib/bmc_redfish_utils.robot 6Resource ../../../lib/logging_utils.robot 7Resource ../../../lib/openbmc_ffdc.robot 8Resource ../../../lib/ipmi_client.robot 9Library ../../../lib/logging_utils.py 10 11Suite Setup Suite Setup Execution 12Suite Teardown Suite Teardown Execution 13Test Setup Test Setup Execution 14Test Teardown Test Teardown Execution 15 16Force Tags Event_Logging 17 18** Variables *** 19 20${max_num_event_logs} ${200} 21 22*** Test Cases *** 23 24Event Log Check After BMC Reboot 25 [Documentation] Check event log after BMC rebooted. 26 [Tags] Event_Log_Check_After_BMC_Reboot 27 28 Redfish Purge Event Log 29 Event Log Should Not Exist 30 31 Redfish OBMC Reboot (off) 32 33 Redfish.Login 34 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 35 36 Event Log Should Not Exist 37 38 39Event Log Check After Host Poweron 40 [Documentation] Check event log after host has booted. 41 [Tags] Event_Log_Check_After_Host_Poweron 42 43 Redfish Purge Event Log 44 Event Log Should Not Exist 45 46 Redfish Power On 47 48 Redfish.Login 49 Event Log Should Not Exist 50 51 52Create Test Event Log And Verify 53 [Documentation] Create event logs and verify via redfish. 54 [Tags] Create_Test_Event_Log_And_Verify 55 56 Create Test Error Log 57 Event Log Should Exist 58 59 60Delete Redfish Event Log And Verify 61 [Documentation] Delete Redfish event log and verify via Redfish. 62 [Tags] Delete_Redfish_Event_Log_And_Verify 63 64 Redfish.Login 65 Redfish Purge Event Log 66 Create Test PEL Log 67 ${elog_entry}= Get Event Logs 68 69 Redfish.Delete /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries/${elog_entry[0]["Id"]} 70 71 ${error_entries}= Get Redfish Error Entries 72 Should Be Empty ${error_entries} 73 74 75Test Event Log Persistency On Restart 76 [Documentation] Restart logging service and verify event logs. 77 [Tags] Test_Event_Log_Persistency_On_Restart 78 79 Create Test Error Log 80 Event Log Should Exist 81 82 BMC Execute Command 83 ... systemctl restart xyz.openbmc_project.Logging.service 84 Sleep 10s reason=Wait for logging service to restart properly. 85 86 Event Log Should Exist 87 88 89Test Event Entry Numbering Reset On Restart 90 [Documentation] Restart logging service and verify event logs entry starts 91 ... from entry "Id" 1. 92 [Tags] Test_Event_Entry_Numbering_Reset_On_Restart 93 [Setup] Redfish Power Off stack_mode=skip 94 95 #{ 96 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 97 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 98 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 99 # "Description": "Collection of System Event Log Entries", 100 # "Members": [ 101 # { 102 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 103 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 104 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 105 # "Created": "2019-05-29T13:19:27+00:00", 106 # "EntryType": "Event", 107 # "Id": "1", <----- Event log ID 108 # "Message": "org.open_power.Host.Error.Event", 109 # "Name": "System DBus Event Log Entry", 110 # "Severity": "Critical" 111 # } 112 # ], 113 # "Members@odata.count": 1, 114 # "Name": "System Event Log Entries" 115 #} 116 117 Create Test PEL Log 118 Create Test PEL Log 119 Event Log Should Exist 120 121 Redfish Purge Event Log 122 Event Log Should Not Exist 123 124 BMC Execute Command 125 ... systemctl restart xyz.openbmc_project.Logging.service 126 Sleep 10s reason=Wait for logging service to restart properly. 127 128 Create Test PEL Log 129 ${elogs}= Get Event Logs 130 131 # After issuing Redfish purge event log, there will be one informational error log 132 # in BMC with ID 1. Due to this the newly generated error log would have ID as 2. 133 Should Be Equal ${elogs[0]["Id"]} 2 msg=Event log entry is not 2 134 135 136Test Event Log Persistency On Reboot 137 [Documentation] Reboot BMC and verify event log. 138 [Tags] Test_Event_Log_Persistency_On_Reboot 139 140 Redfish Purge Event Log 141 Create Test Error Log 142 Event Log Should Exist 143 144 Redfish OBMC Reboot (off) 145 146 Redfish.Login 147 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 148 149 Event Log Should Exist 150 151 152Create Test Event Log And Verify Time Stamp 153 [Documentation] Create event logs and verify time stamp. 154 [Tags] Create_Test_Event_Log_And_Verify_Time_Stamp 155 156 #{ 157 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 158 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 159 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 160 # "Description": "Collection of System Event Log Entries", 161 # "Members": [ 162 # { 163 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 164 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 165 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 166 # "Created": "2023-05-10T10:26:02.186+00:00", <--- Time stamp 167 # "EntryType": "Event", 168 # "Id": "1", 169 # "Message": "org.open_power.Host.Error.Event", 170 # "Name": "System DBus Event Log Entry", 171 # "Severity": "Critical" 172 # } 173 # ], 174 # "Members@odata.count": 1, 175 # "Name": "System Event Log Entries" 176 #} 177 178 Redfish Purge Event Log 179 180 Create Test Error Log 181 Sleep 2s 182 Create Test Error Log 183 184 ${elog_entry}= Get Event Logs 185 186 # The event log generated is associated with the epoc time and unique 187 # for every error and in increasing time stamp. 188 ${time_stamp1}= Convert Date ${elog_entry[0]["Created"].split('.')[0]} epoch 189 ${time_stamp2}= Convert Date ${elog_entry[1]["Created"].split('.')[0]} epoch 190 191 Should Be True ${time_stamp2} > ${time_stamp1} 192 193 194# TODO: openbmc/openbmc-test-automation#1789 195Verify Setting Error Log As Resolved 196 [Documentation] Verify modified field of error log is updated when error log is marked resolved. 197 [Tags] Verify_Setting_Error_Log_As_Resolved 198 199 Create Test PEL Log 200 ${elog_entry}= Get Event Logs 201 202 # Wait for 5 seconds after creating error log. 203 Sleep 5s 204 205 # Mark error log as resolved by setting it to true. 206 Redfish.Patch ${EVENT_LOG_URI}Entries/${elog_entry[0]["Id"]} body={'Resolved':True} 207 208 ${elog_entry}= Get Event Logs 209 210 # Example error log with resolve field set to true: 211 # { 212 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/2045", 213 # "@odata.type": "#LogEntry.v1_8_0.LogEntry", 214 # "AdditionalDataURI": "/redfish/v1/Systems/system/LogServices/EventLog/attachment/2045", 215 # "Created": "2023-05-10T10:26:02.186+00:00", 216 # "EntryType": "Event", 217 # "Id": "2045", 218 # "Message": "xyz.openbmc_project.Host.Error.Event", 219 # "Modified": "2023-05-10T10:26:02.186+00:00", 220 # "Name": "System Event Log Entry", 221 # "Resolved": true, 222 # "Severity": "OK" 223 # } 224 225 Should Be Equal As Strings ${elog_entry[0]["Resolved"]} True 226 227 # Difference created and modified time of error log should be around 5 seconds. 228 ${creation_time}= Convert Date ${elog_entry[0]["Created"].split('.')[0]} epoch 229 ${modification_time}= Convert Date ${elog_entry[0]["Modified"].split('.')[0]} epoch 230 231 ${diff}= Subtract Date From Date ${modification_time} ${creation_time} 232 ${diff}= Convert To Number ${diff} 233 Should Be True 4 < ${diff} < 8 234 235 236Verify IPMI SEL Delete 237 [Documentation] Verify IPMI SEL delete operation. 238 [Tags] Verify_IPMI_SEL_Delete 239 240 Redfish Purge Event Log 241 Create Test Error Log 242 243 ${sel_list}= Run IPMI Standard Command sel list 244 Should Not Be Equal As Strings ${sel_list} SEL has no entries 245 246 # Example of SEL List: 247 # 4 | 04/21/2017 | 10:51:16 | System Event #0x01 | Undetermined system hardware failure | Asserted 248 249 ${sel_entry}= Fetch from Left ${sel_list} | 250 ${sel_entry}= Evaluate $sel_entry.replace(' ','') 251 ${sel_entry}= Convert To Integer 0x${sel_entry} 252 253 ${sel_delete}= Run IPMI Standard Command sel delete ${sel_entry} 254 Should Be Equal As Strings ${sel_delete} Deleted entry ${sel_entry} 255 ... case_insensitive=True 256 257 ${sel_list}= Run IPMI Standard Command sel list 258 Should Be Equal As Strings ${sel_list} SEL has no entries 259 ... case_insensitive=True 260 261 262Verify IPMI SEL Event Entries 263 [Documentation] Verify IPMI SEL's entries info. 264 [Tags] Verify_IPMI_SEL_Event_Entries 265 266 # Generate error logs of random count. 267 ${count}= Evaluate random.randint(1, 5) modules=random 268 Repeat Keyword ${count} Create Test Error Log 269 270 ${sel_entries_count}= Get IPMI SEL Setting Entries 271 Should Be Equal As Strings ${sel_entries_count} ${count} 272 273 274Create Test Event Log And Delete 275 [Documentation] Create an event log and delete it. 276 [Tags] Create_Test_Event_Log_And_Delete 277 278 Create Test Error Log 279 Redfish Purge Event Log 280 Event Log Should Not Exist 281 282 283Create Multiple Test Event Logs And Delete All 284 [Documentation] Create multiple event logs and delete all. 285 [Tags] Create_Multiple_Test_Event_Logs_And_Delete_All 286 287 Create Test Error Log 288 Create Test Error Log 289 Create Test Error Log 290 Redfish Purge Event Log 291 Event Log Should Not Exist 292 293 294Create Two Test Event Logs And Delete One 295 [Documentation] Create two event logs and delete the first entry. 296 [Tags] Create_Two_Test_Event_Logs_And_Delete_One 297 [Setup] Redfish Power Off stack_mode=skip 298 299 Redfish Purge Event Log 300 Create Test PEL Log 301 Create Test PEL Log 302 ${error_entries_before}= Get Redfish Error Entries 303 Redfish.Delete /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries/${error_entries_before[0]} 304 305 ${error_entries_after}= Get Redfish Error Entries 306 Should Not Contain ${error_entries_after} ${error_entries_before[0]} 307 Should Contain ${error_entries_after} ${error_entries_before[1]} 308 309 310Verify Watchdog Timedout Event 311 [Documentation] Trigger watchdog timed out and verify event log generated. 312 [Tags] Verify_Watchdog_Timedout_Event 313 [Teardown] Run Keywords Test Teardown Execution AND Redfish Power Off stack_mode=skip 314 315 Redfish Power Off stack_mode=skip 316 317 # Clear errors if there are any. 318 Redfish.Login 319 Redfish Purge Event Log 320 321 # Reference: [Old legacy REST code] Trigger Host Watchdog Error 322 # Currently, no known redfish interface to set to trigger watchdog timer. 323 324 Redfish Initiate Auto Reboot 1000 325 326 # Logging takes time to generate the timeout error. 327 Wait Until Keyword Succeeds 3 min 20 sec Verify Watchdog EventLog Content 328 329 330Verify Event Logs Capping 331 [Documentation] Verify event logs capping. 332 [Tags] Verify_Event_Logs_Capping 333 334 Redfish Purge Event Log 335 336 ${cmd}= Catenate uptime; for i in {1..201}; do /tmp/tarball/bin/logging-test -c 337 ... AutoTestSimple;sleep 1;done; uptime 338 BMC Execute Command ${cmd} 339 340 ${elogs}= Get Event Logs 341 ${count}= Get Length ${elogs} 342 Run Keyword If ${count} > 200 343 ... Fail Error logs created exceeded max capacity 200. 344 345 346Test Event Log Wrapping 347 [Documentation] Verify event log entries wraps when 200 max cap is reached. 348 [Tags] Test_Event_Log_Wrapping 349 350 # Restarting logging service in order to clear logs and get the next log 351 # ID set to 1. 352 BMC Execute Command 353 ... systemctl restart xyz.openbmc_project.Logging.service 354 Sleep 10s reason=Wait for logging service to restart properly. 355 356 # Create ${max_num_event_logs} event logs. 357 ${cmd}= Catenate uptime; for i in {1..${max_num_event_logs}}; do /tmp/tarball/bin/logging-test -c 358 ... AutoTestSimple;sleep 1;done; uptime 359 BMC Execute Command ${cmd} 360 361 # Verify that event logs with IDs 1 and ${max_num_event_logs} exist. 362 ${event_log}= Get Event Logs 363 364 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 365 Rprint Vars log_entries 366 Should Be Equal As Strings ${log_entries[0]["Id"]} 1 367 368 ${log_entries}= Filter Struct ${event_log} [('Id', '${max_num_event_logs}')] 369 Rprint Vars log_entries 370 Should Be Equal As Strings ${log_entries[0]["Id"]} ${max_num_event_logs} 371 372 # Create event log and verify the entry ID, ${max_num_event_logs + 1}. 373 ${next_event_log_id}= Set Variable ${max_num_event_logs + 1} 374 375 Create Test Error Log 376 377 ${event_log}= Get Event Logs 378 379 ${log_entries}= Filter Struct ${event_log} [('Id', '${next_event_log_id}')] 380 Rprint Vars log_entries 381 Should Be Equal As Strings ${log_entries[0]["Id"]} ${next_event_log_id} 382 383 # Event log 1 should be wrapped. 384 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 385 Rprint Vars log_entries 386 387 ${length_log_entries} Get Length ${log_entries} 388 Should Be Equal As Integers ${length_log_entries} 0 389 ... msg=The event log should have wrapped such that entry ID 1 is now purged. 390 391 392Verify Default Value Of Resolved Field Is False For An Error Log Via Redfish 393 [Documentation] Verify the Resolve field status is false for an error log from Redfish. 394 [Tags] Verify_Default_Value_Of_Resolved_Field_Is_False_For_An_Error_Log_Via_Redfish 395 396 Redfish Purge Event Log 397 Create Test Error Log 398 399 # Check resolve field value of created error log. 400 ${elog_entry}= Get Event Logs 401 Should Be Equal ${elog_entry[0]["Resolved"]} ${False} 402 403 404*** Keywords *** 405 406Suite Setup Execution 407 [Documentation] Do test case setup tasks. 408 409 Redfish.Login 410 411 Redfish Purge Event Log 412 413 ${status}= Run Keyword And Return Status Logging Test Binary Exist 414 Run Keyword If ${status} == ${False} Install Tarball 415 416 417Suite Teardown Execution 418 [Documentation] Do the post suite teardown. 419 420 Redfish.Logout 421 422 423Test Setup Execution 424 [Documentation] Do test case setup tasks. 425 426 Redfish Purge Event Log 427 428 ${status}= Run Keyword And Return Status Logging Test Binary Exist 429 Run Keyword If ${status} == ${False} Install Tarball 430 431 432Test Teardown Execution 433 [Documentation] Do the post test teardown. 434 435 FFDC On Test Case Fail 436 Redfish.Login 437 Redfish Purge Event Log 438 439 440Get Redfish Error Entries 441 [Documentation] Return Redfish error ids list. 442 ${error_uris}= redfish_utils.get_member_list /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries 443 ${error_ids}= Create List 444 445 FOR ${error_uri} IN @{error_uris} 446 ${error_id}= Fetch From Right ${error_uri} / 447 Append To List ${error_ids} ${error_id} 448 END 449 450 [Return] ${error_ids} 451 452 453Event Log Should Not Exist 454 [Documentation] Event log entries should not exist. 455 456 ${elogs}= Get Event Logs 457 Should Be Empty ${elogs} msg=System event log entry is not empty. 458 459 460Event Log Should Exist 461 [Documentation] Event log entries should exist. 462 463 ${elogs}= Get Event Logs 464 Should Not Be Empty ${elogs} msg=System event log entry is not empty. 465 466 467Verify Watchdog EventLog Content 468 [Documentation] Verify watchdog event log content. 469 470 # Example: 471 # { 472 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 473 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/31", 474 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 475 # "Created": "2019-05-31T18:41:33+00:00", 476 # "EntryType": "Event", 477 # "Id": "31", 478 # "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut", 479 # "Name": "System DBus Event Log Entry", 480 # "Severity": "Critical" 481 # } 482 483 ${elog_list}= Get Event Logs 484 485 Rprint Vars elog_list 486 487 FOR ${entry} IN @{elog_list} 488 ${found_match}= Run Keyword And Return Status Is Watchdog Error Found ${entry} 489 Exit For Loop If '${found_match}' == 'True' 490 END 491 492 Run Keyword If '${found_match}' == 'False' Fail msg=No watchdog error logged. 493 494 495Is Watchdog Error Found 496 [Documentation] Check if the give log entry matches specific watchdog error. 497 [Arguments] ${elog} 498 499 # Description of argument(s): 500 # elog Error log entry dictionary data. 501 502 Should Contain Any 503 ... ${elog["Message"]} org.open_power.Host.Boot.Error.WatchdogTimedOut 504 ... CEC Hardware - Hostboot-Service Processor Interface 505 ... msg=Watchdog timeout event log was not found. 506 507 Log To Console Matched Found: ${elog} 508