1*** Settings *** 2Documentation Inventory of hardware resources under systems. 3 4Resource ../../../lib/bmc_redfish_resource.robot 5Resource ../../../lib/bmc_redfish_utils.robot 6Resource ../../../lib/logging_utils.robot 7Resource ../../../lib/openbmc_ffdc.robot 8Resource ../../../lib/ipmi_client.robot 9Library ../../../lib/logging_utils.py 10 11Suite Setup Suite Setup Execution 12Suite Teardown Suite Teardown Execution 13Test Setup Test Setup Execution 14Test Teardown Test Teardown Execution 15 16** Variables *** 17 18${max_num_event_logs} ${200} 19 20*** Test Cases *** 21 22Event Log Check After BMC Reboot 23 [Documentation] Check event log after BMC rebooted. 24 [Tags] Event_Log_Check_After_BMC_Reboot 25 26 Redfish Purge Event Log 27 Event Log Should Not Exist 28 29 Redfish OBMC Reboot (off) 30 31 Redfish.Login 32 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 33 34 Event Log Should Not Exist 35 36 37Event Log Check After Host Poweron 38 [Documentation] Check event log after host has booted. 39 [Tags] Event_Log_Check_After_Host_Poweron 40 41 Redfish Purge Event Log 42 Event Log Should Not Exist 43 44 Redfish Power On 45 46 Redfish.Login 47 Event Log Should Not Exist 48 49 50Create Test Event Log And Verify 51 [Documentation] Create event logs and verify via redfish. 52 [Tags] Create_Test_Event_Log_And_Verify 53 54 Create Test Error Log 55 Event Log Should Exist 56 57 58Delete Redfish Event Log And Verify 59 [Documentation] Delete Redfish event log and verify via Redfish. 60 [Tags] Delete_Redfish_Event_Log_And_Verify 61 62 Redfish.Login 63 Redfish Purge Event Log 64 Create Test PEL Log 65 ${elog_entry}= Get Event Logs 66 67 Redfish.Delete /redfish/v1/Systems/system/LogServices/EventLog/Entries/${elog_entry[0]["Id"]} 68 69 ${error_entries}= Get Redfish Error Entries 70 Should Be Empty ${error_entries} 71 72 73Test Event Log Persistency On Restart 74 [Documentation] Restart logging service and verify event logs. 75 [Tags] Test_Event_Log_Persistency_On_Restart 76 77 Create Test Error Log 78 Event Log Should Exist 79 80 BMC Execute Command 81 ... systemctl restart xyz.openbmc_project.Logging.service 82 Sleep 10s reason=Wait for logging service to restart properly. 83 84 Event Log Should Exist 85 86 87Test Event Entry Numbering Reset On Restart 88 [Documentation] Restart logging service and verify event logs entry starts 89 ... from entry "Id" 1. 90 [Tags] Test_Event_Entry_Numbering_Reset_On_Restart 91 92 #{ 93 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 94 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 95 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 96 # "Description": "Collection of System Event Log Entries", 97 # "Members": [ 98 # { 99 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 100 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 101 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 102 # "Created": "2019-05-29T13:19:27+00:00", 103 # "EntryType": "Event", 104 # "Id": "1", <----- Event log ID 105 # "Message": "org.open_power.Host.Error.Event", 106 # "Name": "System DBus Event Log Entry", 107 # "Severity": "Critical" 108 # } 109 # ], 110 # "Members@odata.count": 1, 111 # "Name": "System Event Log Entries" 112 #} 113 114 Create Test Error Log 115 Create Test Error Log 116 Event Log Should Exist 117 118 Redfish Purge Event Log 119 Event Log Should Not Exist 120 121 BMC Execute Command 122 ... systemctl restart xyz.openbmc_project.Logging.service 123 Sleep 10s reason=Wait for logging service to restart properly. 124 125 Create Test Error Log 126 ${elogs}= Get Event Logs 127 Should Be Equal ${elogs[0]["Id"]} 1 msg=Event log entry is not 1. 128 129 130Test Event Log Persistency On Reboot 131 [Documentation] Reboot BMC and verify event log. 132 [Tags] Test_Event_Log_Persistency_On_Reboot 133 134 Redfish Purge Event Log 135 Create Test Error Log 136 Event Log Should Exist 137 138 Redfish OBMC Reboot (off) 139 140 Redfish.Login 141 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 142 143 Event Log Should Exist 144 145 146# TODO: openbmc/openbmc-test-automation#1789 147Create Test Event Log And Verify Resolved Field 148 [Documentation] Create event log and verify "Resolved" field is 0. 149 [Tags] Create_Test_Event_Log_And_Verify_Resolved_Field 150 151 # Example Error log: 152 # "/xyz/openbmc_project/logging/entry/1": { 153 # "AdditionalData": [ 154 # "STRING=FOO" 155 # ], 156 # "Id": 1, 157 # "Message": "example.xyz.openbmc_project.Example.Elog.AutoTestSimple", 158 # "Resolved": 0, 159 # "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error", 160 # "Timestamp": 1490817164983, 161 # "Associations": [] 162 # }, 163 164 # To mark an error as resolved, without deleting the error, user would 165 # set this bool property. 166 # In this test context we are making sure "Resolved" field is "0" 167 # by default. 168 169 Redfish Purge Event Log 170 Create Test Error Log 171 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 172 ${resolved}= Read Attribute ${elog_entry[0]} Resolved 173 Should Be True ${resolved} == 0 174 175 176Create Test Event Log And Verify Time Stamp 177 [Documentation] Create event logs and verify time stamp. 178 [Tags] Create_Test_Event_Log_And_Verify_Time_Stamp 179 180 #{ 181 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 182 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 183 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 184 # "Description": "Collection of System Event Log Entries", 185 # "Members": [ 186 # { 187 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 188 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 189 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 190 # "Created": "2019-05-29T13:19:27+00:00", <--- Time stamp 191 # "EntryType": "Event", 192 # "Id": "1", 193 # "Message": "org.open_power.Host.Error.Event", 194 # "Name": "System DBus Event Log Entry", 195 # "Severity": "Critical" 196 # } 197 # ], 198 # "Members@odata.count": 1, 199 # "Name": "System Event Log Entries" 200 #} 201 202 Redfish Purge Event Log 203 204 Create Test Error Log 205 Sleep 2s 206 Create Test Error Log 207 208 ${elog_entry}= Get Event Logs 209 210 # The event log generated is associated with the epoc time and unique 211 # for every error and in increasing time stamp. 212 ${time_stamp1}= Convert Date ${elog_entry[0]["Created"]} epoch 213 ${time_stamp2}= Convert Date ${elog_entry[1]["Created"]} epoch 214 215 Should Be True ${time_stamp2} > ${time_stamp1} 216 217 218Verify Setting Error Log As Resolved 219 [Documentation] Verify modified field of error log is updated when error log is marked resolved. 220 [Tags] Verify_Setting_Error_Log_As_Resolved 221 222 Create Test PEL Log 223 ${elog_entry}= Get Event Logs 224 225 # Wait for 5 seconds after creating error log. 226 Sleep 5s 227 228 # Mark error log as resolved by setting it to true. 229 Redfish.Patch ${EVENT_LOG_URI}Entries/${elog_entry[0]["Id"]} body={'Resolved':True} 230 231 ${elog_entry}= Get Event Logs 232 233 # Example error log with resolve field set to true: 234 # { 235 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/2045", 236 # "@odata.type": "#LogEntry.v1_8_0.LogEntry", 237 # "AdditionalDataURI": "/redfish/v1/Systems/system/LogServices/EventLog/attachment/2045", 238 # "Created": "2021-05-11T04:45:07+00:00", 239 # "EntryType": "Event", 240 # "Id": "2045", 241 # "Message": "xyz.openbmc_project.Host.Error.Event", 242 # "Modified": "2021-05-11T07:24:36+00:00", 243 # "Name": "System Event Log Entry", 244 # "Resolved": true, 245 # "Severity": "OK" 246 # } 247 248 Should Be Equal As Strings ${elog_entry[0]["Resolved"]} True 249 250 # Difference created and modified time of error log should be around 5 seconds. 251 ${creation_time}= Convert Date ${elog_entry[0]["Created"]} epoch 252 ${modification_time}= Convert Date ${elog_entry[0]["Modified"]} epoch 253 254 ${diff}= Subtract Date From Date ${modification_time} ${creation_time} 255 ${diff}= Convert To Number ${diff} 256 Should Be True 4 < ${diff} < 8 257 258 259Verify IPMI SEL Delete 260 [Documentation] Verify IPMI SEL delete operation. 261 [Tags] Verify_IPMI_SEL_Delete 262 263 Redfish Purge Event Log 264 Create Test Error Log 265 266 ${sel_list}= Run IPMI Standard Command sel list 267 Should Not Be Equal As Strings ${sel_list} SEL has no entries 268 269 # Example of SEL List: 270 # 4 | 04/21/2017 | 10:51:16 | System Event #0x01 | Undetermined system hardware failure | Asserted 271 272 ${sel_entry}= Fetch from Left ${sel_list} | 273 ${sel_entry}= Evaluate $sel_entry.replace(' ','') 274 ${sel_entry}= Convert To Integer 0x${sel_entry} 275 276 ${sel_delete}= Run IPMI Standard Command sel delete ${sel_entry} 277 Should Be Equal As Strings ${sel_delete} Deleted entry ${sel_entry} 278 ... case_insensitive=True 279 280 ${sel_list}= Run IPMI Standard Command sel list 281 Should Be Equal As Strings ${sel_list} SEL has no entries 282 ... case_insensitive=True 283 284 285Delete Non Existing SEL Event Entry 286 [Documentation] Delete non existing SEL event entry. 287 [Tags] Delete_Non_Existing_SEL_Event_Entry 288 289 ${sel_delete}= Run Keyword And Expect Error * 290 ... Run IPMI Standard Command sel delete 100 291 Should Contain ${sel_delete} Unable to delete entry 292 ... case_insensitive=True 293 294 295Delete Invalid SEL Event Entry 296 [Documentation] Delete invalid SEL event entry. 297 [Tags] Delete_Invalid_SEL_Event_Entry 298 299 ${sel_delete}= Run Keyword And Expect Error * 300 ... Run IPMI Standard Command sel delete abc 301 Should Contain ${sel_delete} Given SEL ID 'abc' is invalid 302 ... case_insensitive=True 303 304 305Verify IPMI SEL Event Entries 306 [Documentation] Verify IPMI SEL's entries info. 307 [Tags] Verify_IPMI_SEL_Event_Entries 308 309 # Generate error logs of random count. 310 ${count}= Evaluate random.randint(1, 5) modules=random 311 Repeat Keyword ${count} Create Test Error Log 312 313 ${sel_entries_count}= Get IPMI SEL Setting Entries 314 Should Be Equal As Strings ${sel_entries_count} ${count} 315 316 317Verify IPMI SEL Event Last Add Time 318 [Documentation] Verify IPMI SEL's last added timestamp. 319 [Tags] Verify_IPMI_SEL_Event_Last_Add_Time 320 321 Create Test Error Log 322 ${sel_time}= Run IPMI Standard Command sel time get 323 ${sel_time}= Convert Date ${sel_time} 324 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 325 326 ${sel_last_add_time}= Get IPMI SEL Setting Last Add Time 327 ${sel_last_add_time}= Convert Date ${sel_last_add_time} 328 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 329 330 ${time_diff}= 331 ... Subtract Date From Date ${sel_last_add_time} ${sel_time} 332 333 # Verify if the delay in current time check and last add SEL time 334 # is less or equals to 2 seconds. 335 Should Be True ${time_diff} <= 2 336 337 338Create Test Event Log And Delete 339 [Documentation] Create an event log and delete it. 340 [Tags] Create_Test_Event_Log_And_Delete 341 342 Create Test Error Log 343 Redfish Purge Event Log 344 Event Log Should Not Exist 345 346 347Create Multiple Test Event Logs And Delete All 348 [Documentation] Create multiple event logs and delete all. 349 [Tags] Create_Multiple_Test_Event_Logs_And_Delete_All 350 351 Create Test Error Log 352 Create Test Error Log 353 Create Test Error Log 354 Redfish Purge Event Log 355 Event Log Should Not Exist 356 357 358# TODO: openbmc/openbmc-test-automation#1789 359Create Two Test Event Logs And Delete One 360 [Documentation] Create two event logs and delete the first entry. 361 [Tags] Create_Two_Test_Event_Logs_And_Delete_One 362 363 Redfish Purge Event Log 364 Create Test Error Log 365 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 366 Create Test Error Log 367 Delete Error log Entry ${elog_entry[0]} 368 ${resp}= OpenBMC Get Request ${elog_entry[0]} 369 Should Be Equal As Strings ${resp.status_code} ${HTTP_NOT_FOUND} 370 371 372Verify Watchdog Timedout Event 373 [Documentation] Trigger watchdog timed out and verify event log generated. 374 [Tags] Verify_Watchdog_Timedout_Event 375 376 Redfish Power On 377 378 # Clear errors if there are any. 379 Redfish.Login 380 Redfish Purge Event Log 381 382 Trigger Host Watchdog Error 383 384 # Logging takes time to generate the timeout error. 385 Wait Until Keyword Succeeds 2 min 30 sec 386 ... Verify Watchdog EventLog Content 387 388 389Verify Event Logs Capping 390 [Documentation] Verify event logs capping. 391 [Tags] Verify_Event_Logs_Capping 392 393 Redfish Purge Event Log 394 395 ${cmd}= Catenate for i in {1..201}; do /tmp/tarball/bin/logging-test -c 396 ... AutoTestSimple;sleep 1;done 397 BMC Execute Command ${cmd} 398 399 ${elogs}= Get Event Logs 400 ${count}= Get Length ${elogs} 401 Run Keyword If ${count} > 200 402 ... Fail Error logs created exceeded max capacity 200. 403 404 405Test Event Log Wrapping 406 [Documentation] Verify event log entries wraps when 200 max cap is reached. 407 [Tags] Test_Event_Log_Wrapping 408 409 # Restarting logging service in order to clear logs and get the next log 410 # ID set to 1. 411 BMC Execute Command 412 ... systemctl restart xyz.openbmc_project.Logging.service 413 Sleep 10s reason=Wait for logging service to restart properly. 414 415 # Create ${max_num_event_logs} event logs. 416 ${cmd}= Catenate for i in {1..${max_num_event_logs}}; do /tmp/tarball/bin/logging-test -c 417 ... AutoTestSimple;sleep 1;done 418 BMC Execute Command ${cmd} 419 420 # Verify that event logs with IDs 1 and ${max_num_event_logs} exist. 421 ${event_log}= Get Event Logs 422 423 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 424 Rprint Vars log_entries 425 Should Be Equal As Strings ${log_entries[0]["Id"]} 1 426 427 ${log_entries}= Filter Struct ${event_log} [('Id', '${max_num_event_logs}')] 428 Rprint Vars log_entries 429 Should Be Equal As Strings ${log_entries[0]["Id"]} ${max_num_event_logs} 430 431 # Create event log and verify the entry ID, ${max_num_event_logs + 1}. 432 ${next_event_log_id}= Set Variable ${max_num_event_logs + 1} 433 434 Create Test Error Log 435 436 ${event_log}= Get Event Logs 437 438 ${log_entries}= Filter Struct ${event_log} [('Id', '${next_event_log_id}')] 439 Rprint Vars log_entries 440 Should Be Equal As Strings ${log_entries[0]["Id"]} ${next_event_log_id} 441 442 # Event log 1 should be wrapped. 443 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 444 Rprint Vars log_entries 445 446 ${length_log_entries} Get Length ${log_entries} 447 Should Be Equal As Integers ${length_log_entries} 0 448 ... msg=The event log should have wrapped such that entry ID 1 is now purged. 449 450 451*** Keywords *** 452 453Suite Setup Execution 454 [Documentation] Do test case setup tasks. 455 456 Redfish.Login 457 458 Redfish Purge Event Log 459 460 ${status}= Run Keyword And Return Status Logging Test Binary Exist 461 Run Keyword If ${status} == ${False} Install Tarball 462 463 464Suite Teardown Execution 465 [Documentation] Do the post suite teardown. 466 467 Redfish.Logout 468 469 470Test Setup Execution 471 [Documentation] Do test case setup tasks. 472 473 Redfish Purge Event Log 474 475 ${status}= Run Keyword And Return Status Logging Test Binary Exist 476 Run Keyword If ${status} == ${False} Install Tarball 477 478 479Test Teardown Execution 480 [Documentation] Do the post test teardown. 481 482 FFDC On Test Case Fail 483 Redfish.Login 484 Redfish Purge Event Log 485 486 487Get Redfish Error Entries 488 [Documentation] Return Redfish error ids list. 489 ${error_uris}= redfish_utils.get_member_list /redfish/v1/Systems/system/LogServices/EventLog/Entries 490 ${error_ids}= Create List 491 492 FOR ${error_uri} IN @{error_uris} 493 ${error_id}= Fetch From Right ${error_uri} / 494 Append To List ${error_ids} ${error_id} 495 END 496 497 [Return] ${error_ids} 498 499 500Event Log Should Not Exist 501 [Documentation] Event log entries should not exist. 502 503 ${elogs}= Get Event Logs 504 Should Be Empty ${elogs} msg=System event log entry is not empty. 505 506 507Event Log Should Exist 508 [Documentation] Event log entries should exist. 509 510 ${elogs}= Get Event Logs 511 Should Not Be Empty ${elogs} msg=System event log entry is not empty. 512 513 514Verify Watchdog EventLog Content 515 [Documentation] Verify watchdog event log content. 516 517 # Example: 518 # { 519 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 520 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/31", 521 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 522 # "Created": "2019-05-31T18:41:33+00:00", 523 # "EntryType": "Event", 524 # "Id": "31", 525 # "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut", 526 # "Name": "System DBus Event Log Entry", 527 # "Severity": "Critical" 528 # } 529 530 ${elog}= Get Event Logs 531 Should Be Equal As Strings 532 ... ${elog[0]["Message"]} org.open_power.Host.Boot.Error.WatchdogTimedOut 533 ... msg=Watchdog timeout event log was not found. 534 Should Be Equal As Strings 535 ... ${elog[0]["Severity"]} Critical 536 ... msg=Watchdog timeout severity unexpected value. 537