1*** Settings *** 2Documentation Inventory of hardware resources under systems. 3 4Resource ../../../lib/bmc_redfish_resource.robot 5Resource ../../../lib/bmc_redfish_utils.robot 6Resource ../../../lib/logging_utils.robot 7Resource ../../../lib/openbmc_ffdc.robot 8Resource ../../../lib/ipmi_client.robot 9Library ../../../lib/logging_utils.py 10 11Suite Setup Suite Setup Execution 12Suite Teardown Suite Teardown Execution 13Test Setup Test Setup Execution 14Test Teardown Test Teardown Execution 15 16** Variables *** 17 18${max_num_event_logs} ${200} 19 20*** Test Cases *** 21 22Event Log Check After BMC Reboot 23 [Documentation] Check event log after BMC rebooted. 24 [Tags] Event_Log_Check_After_BMC_Reboot 25 26 Redfish Purge Event Log 27 Event Log Should Not Exist 28 29 Redfish OBMC Reboot (off) 30 31 Redfish.Login 32 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 33 34 Event Log Should Not Exist 35 36 37Event Log Check After Host Poweron 38 [Documentation] Check event log after host has booted. 39 [Tags] Event_Log_Check_After_Host_Poweron 40 41 Redfish Purge Event Log 42 Event Log Should Not Exist 43 44 Redfish Power On 45 46 Redfish.Login 47 Event Log Should Not Exist 48 49 50Create Test Event Log And Verify 51 [Documentation] Create event logs and verify via redfish. 52 [Tags] Create_Test_Event_Log_And_Verify 53 54 Create Test Error Log 55 Event Log Should Exist 56 57 58Delete Redfish Event Log And Verify 59 [Documentation] Delete Redfish event log and verify via Redfish. 60 [Tags] Delete_Redfish_Event_Log_And_Verify 61 62 Redfish.Login 63 Redfish Purge Event Log 64 Create Test PEL Log 65 ${elog_entry}= Get Event Logs 66 67 Redfish.Delete /redfish/v1/Systems/system/LogServices/EventLog/Entries/${elog_entry[0]["Id"]} 68 69 ${error_entries}= Get Redfish Error Entries 70 Should Be Empty ${error_entries} 71 72 73Test Event Log Persistency On Restart 74 [Documentation] Restart logging service and verify event logs. 75 [Tags] Test_Event_Log_Persistency_On_Restart 76 77 Create Test Error Log 78 Event Log Should Exist 79 80 BMC Execute Command 81 ... systemctl restart xyz.openbmc_project.Logging.service 82 Sleep 10s reason=Wait for logging service to restart properly. 83 84 Event Log Should Exist 85 86 87Test Event Entry Numbering Reset On Restart 88 [Documentation] Restart logging service and verify event logs entry starts 89 ... from entry "Id" 1. 90 [Tags] Test_Event_Entry_Numbering_Reset_On_Restart 91 92 #{ 93 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 94 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 95 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 96 # "Description": "Collection of System Event Log Entries", 97 # "Members": [ 98 # { 99 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 100 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 101 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 102 # "Created": "2019-05-29T13:19:27+00:00", 103 # "EntryType": "Event", 104 # "Id": "1", <----- Event log ID 105 # "Message": "org.open_power.Host.Error.Event", 106 # "Name": "System DBus Event Log Entry", 107 # "Severity": "Critical" 108 # } 109 # ], 110 # "Members@odata.count": 1, 111 # "Name": "System Event Log Entries" 112 #} 113 114 Create Test Error Log 115 Create Test Error Log 116 Event Log Should Exist 117 118 Redfish Purge Event Log 119 Event Log Should Not Exist 120 121 BMC Execute Command 122 ... systemctl restart xyz.openbmc_project.Logging.service 123 Sleep 10s reason=Wait for logging service to restart properly. 124 125 Create Test Error Log 126 ${elogs}= Get Event Logs 127 Should Be Equal ${elogs[0]["Id"]} 1 msg=Event log entry is not 1. 128 129 130Test Event Log Persistency On Reboot 131 [Documentation] Reboot BMC and verify event log. 132 [Tags] Test_Event_Log_Persistency_On_Reboot 133 134 Redfish Purge Event Log 135 Create Test Error Log 136 Event Log Should Exist 137 138 Redfish OBMC Reboot (off) 139 140 Redfish.Login 141 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 142 143 Event Log Should Exist 144 145 146Create Test Event Log And Verify Time Stamp 147 [Documentation] Create event logs and verify time stamp. 148 [Tags] Create_Test_Event_Log_And_Verify_Time_Stamp 149 150 #{ 151 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 152 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 153 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 154 # "Description": "Collection of System Event Log Entries", 155 # "Members": [ 156 # { 157 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 158 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 159 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 160 # "Created": "2019-05-29T13:19:27+00:00", <--- Time stamp 161 # "EntryType": "Event", 162 # "Id": "1", 163 # "Message": "org.open_power.Host.Error.Event", 164 # "Name": "System DBus Event Log Entry", 165 # "Severity": "Critical" 166 # } 167 # ], 168 # "Members@odata.count": 1, 169 # "Name": "System Event Log Entries" 170 #} 171 172 Redfish Purge Event Log 173 174 Create Test Error Log 175 Sleep 2s 176 Create Test Error Log 177 178 ${elog_entry}= Get Event Logs 179 180 # The event log generated is associated with the epoc time and unique 181 # for every error and in increasing time stamp. 182 ${time_stamp1}= Convert Date ${elog_entry[0]["Created"]} epoch 183 ${time_stamp2}= Convert Date ${elog_entry[1]["Created"]} epoch 184 185 Should Be True ${time_stamp2} > ${time_stamp1} 186 187 188# TODO: openbmc/openbmc-test-automation#1789 189Verify Setting Error Log As Resolved 190 [Documentation] Verify modified field of error log is updated when error log is marked resolved. 191 [Tags] Verify_Setting_Error_Log_As_Resolved 192 193 Create Test PEL Log 194 ${elog_entry}= Get Event Logs 195 196 # Wait for 5 seconds after creating error log. 197 Sleep 5s 198 199 # Mark error log as resolved by setting it to true. 200 Redfish.Patch ${EVENT_LOG_URI}Entries/${elog_entry[0]["Id"]} body={'Resolved':True} 201 202 ${elog_entry}= Get Event Logs 203 204 # Example error log with resolve field set to true: 205 # { 206 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/2045", 207 # "@odata.type": "#LogEntry.v1_8_0.LogEntry", 208 # "AdditionalDataURI": "/redfish/v1/Systems/system/LogServices/EventLog/attachment/2045", 209 # "Created": "2021-05-11T04:45:07+00:00", 210 # "EntryType": "Event", 211 # "Id": "2045", 212 # "Message": "xyz.openbmc_project.Host.Error.Event", 213 # "Modified": "2021-05-11T07:24:36+00:00", 214 # "Name": "System Event Log Entry", 215 # "Resolved": true, 216 # "Severity": "OK" 217 # } 218 219 Should Be Equal As Strings ${elog_entry[0]["Resolved"]} True 220 221 # Difference created and modified time of error log should be around 5 seconds. 222 ${creation_time}= Convert Date ${elog_entry[0]["Created"]} epoch 223 ${modification_time}= Convert Date ${elog_entry[0]["Modified"]} epoch 224 225 ${diff}= Subtract Date From Date ${modification_time} ${creation_time} 226 ${diff}= Convert To Number ${diff} 227 Should Be True 4 < ${diff} < 8 228 229 230Verify IPMI SEL Delete 231 [Documentation] Verify IPMI SEL delete operation. 232 [Tags] Verify_IPMI_SEL_Delete 233 234 Redfish Purge Event Log 235 Create Test Error Log 236 237 ${sel_list}= Run IPMI Standard Command sel list 238 Should Not Be Equal As Strings ${sel_list} SEL has no entries 239 240 # Example of SEL List: 241 # 4 | 04/21/2017 | 10:51:16 | System Event #0x01 | Undetermined system hardware failure | Asserted 242 243 ${sel_entry}= Fetch from Left ${sel_list} | 244 ${sel_entry}= Evaluate $sel_entry.replace(' ','') 245 ${sel_entry}= Convert To Integer 0x${sel_entry} 246 247 ${sel_delete}= Run IPMI Standard Command sel delete ${sel_entry} 248 Should Be Equal As Strings ${sel_delete} Deleted entry ${sel_entry} 249 ... case_insensitive=True 250 251 ${sel_list}= Run IPMI Standard Command sel list 252 Should Be Equal As Strings ${sel_list} SEL has no entries 253 ... case_insensitive=True 254 255 256Delete Non Existing SEL Event Entry 257 [Documentation] Delete non existing SEL event entry. 258 [Tags] Delete_Non_Existing_SEL_Event_Entry 259 260 ${sel_delete}= Run Keyword And Expect Error * 261 ... Run IPMI Standard Command sel delete 100 262 Should Contain ${sel_delete} Unable to delete entry 263 ... case_insensitive=True 264 265 266Delete Invalid SEL Event Entry 267 [Documentation] Delete invalid SEL event entry. 268 [Tags] Delete_Invalid_SEL_Event_Entry 269 270 ${sel_delete}= Run Keyword And Expect Error * 271 ... Run IPMI Standard Command sel delete abc 272 Should Contain ${sel_delete} Given SEL ID 'abc' is invalid 273 ... case_insensitive=True 274 275 276Verify IPMI SEL Event Entries 277 [Documentation] Verify IPMI SEL's entries info. 278 [Tags] Verify_IPMI_SEL_Event_Entries 279 280 # Generate error logs of random count. 281 ${count}= Evaluate random.randint(1, 5) modules=random 282 Repeat Keyword ${count} Create Test Error Log 283 284 ${sel_entries_count}= Get IPMI SEL Setting Entries 285 Should Be Equal As Strings ${sel_entries_count} ${count} 286 287 288Verify IPMI SEL Event Last Add Time 289 [Documentation] Verify IPMI SEL's last added timestamp. 290 [Tags] Verify_IPMI_SEL_Event_Last_Add_Time 291 292 Create Test Error Log 293 ${sel_time}= Run IPMI Standard Command sel time get 294 ${sel_time}= Convert Date ${sel_time} 295 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 296 297 ${sel_last_add_time}= Get IPMI SEL Setting Last Add Time 298 ${sel_last_add_time}= Convert Date ${sel_last_add_time} 299 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 300 301 ${time_diff}= 302 ... Subtract Date From Date ${sel_last_add_time} ${sel_time} 303 304 # Verify if the delay in current time check and last add SEL time 305 # is less or equals to 2 seconds. 306 Should Be True ${time_diff} <= 2 307 308 309Create Test Event Log And Delete 310 [Documentation] Create an event log and delete it. 311 [Tags] Create_Test_Event_Log_And_Delete 312 313 Create Test Error Log 314 Redfish Purge Event Log 315 Event Log Should Not Exist 316 317 318Create Multiple Test Event Logs And Delete All 319 [Documentation] Create multiple event logs and delete all. 320 [Tags] Create_Multiple_Test_Event_Logs_And_Delete_All 321 322 Create Test Error Log 323 Create Test Error Log 324 Create Test Error Log 325 Redfish Purge Event Log 326 Event Log Should Not Exist 327 328 329Create Two Test Event Logs And Delete One 330 [Documentation] Create two event logs and delete the first entry. 331 [Tags] Create_Two_Test_Event_Logs_And_Delete_One 332 [Setup] Redfish Power Off stack_mode=skip 333 334 Redfish Purge Event Log 335 Create Test PEL Log 336 Create Test PEL Log 337 ${error_entries_before}= Get Redfish Error Entries 338 Redfish.Delete /redfish/v1/Systems/system/LogServices/EventLog/Entries/${error_entries_before[0]} 339 340 ${error_entries_after}= Get Redfish Error Entries 341 Should Not Contain ${error_entries_after} ${error_entries_before[0]} 342 Should Contain ${error_entries_after} ${error_entries_before[1]} 343 344 345Verify Watchdog Timedout Event 346 [Documentation] Trigger watchdog timed out and verify event log generated. 347 [Tags] Verify_Watchdog_Timedout_Event 348 349 Redfish Power On 350 351 # Clear errors if there are any. 352 Redfish.Login 353 Redfish Purge Event Log 354 355 Trigger Host Watchdog Error 356 357 # Logging takes time to generate the timeout error. 358 Wait Until Keyword Succeeds 2 min 30 sec 359 ... Verify Watchdog EventLog Content 360 361 362Verify Event Logs Capping 363 [Documentation] Verify event logs capping. 364 [Tags] Verify_Event_Logs_Capping 365 366 Redfish Purge Event Log 367 368 ${cmd}= Catenate for i in {1..201}; do /tmp/tarball/bin/logging-test -c 369 ... AutoTestSimple;sleep 1;done 370 BMC Execute Command ${cmd} 371 372 ${elogs}= Get Event Logs 373 ${count}= Get Length ${elogs} 374 Run Keyword If ${count} > 200 375 ... Fail Error logs created exceeded max capacity 200. 376 377 378Test Event Log Wrapping 379 [Documentation] Verify event log entries wraps when 200 max cap is reached. 380 [Tags] Test_Event_Log_Wrapping 381 382 # Restarting logging service in order to clear logs and get the next log 383 # ID set to 1. 384 BMC Execute Command 385 ... systemctl restart xyz.openbmc_project.Logging.service 386 Sleep 10s reason=Wait for logging service to restart properly. 387 388 # Create ${max_num_event_logs} event logs. 389 ${cmd}= Catenate for i in {1..${max_num_event_logs}}; do /tmp/tarball/bin/logging-test -c 390 ... AutoTestSimple;sleep 1;done 391 BMC Execute Command ${cmd} 392 393 # Verify that event logs with IDs 1 and ${max_num_event_logs} exist. 394 ${event_log}= Get Event Logs 395 396 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 397 Rprint Vars log_entries 398 Should Be Equal As Strings ${log_entries[0]["Id"]} 1 399 400 ${log_entries}= Filter Struct ${event_log} [('Id', '${max_num_event_logs}')] 401 Rprint Vars log_entries 402 Should Be Equal As Strings ${log_entries[0]["Id"]} ${max_num_event_logs} 403 404 # Create event log and verify the entry ID, ${max_num_event_logs + 1}. 405 ${next_event_log_id}= Set Variable ${max_num_event_logs + 1} 406 407 Create Test Error Log 408 409 ${event_log}= Get Event Logs 410 411 ${log_entries}= Filter Struct ${event_log} [('Id', '${next_event_log_id}')] 412 Rprint Vars log_entries 413 Should Be Equal As Strings ${log_entries[0]["Id"]} ${next_event_log_id} 414 415 # Event log 1 should be wrapped. 416 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 417 Rprint Vars log_entries 418 419 ${length_log_entries} Get Length ${log_entries} 420 Should Be Equal As Integers ${length_log_entries} 0 421 ... msg=The event log should have wrapped such that entry ID 1 is now purged. 422 423 424*** Keywords *** 425 426Suite Setup Execution 427 [Documentation] Do test case setup tasks. 428 429 Redfish.Login 430 431 Redfish Purge Event Log 432 433 ${status}= Run Keyword And Return Status Logging Test Binary Exist 434 Run Keyword If ${status} == ${False} Install Tarball 435 436 437Suite Teardown Execution 438 [Documentation] Do the post suite teardown. 439 440 Redfish.Logout 441 442 443Test Setup Execution 444 [Documentation] Do test case setup tasks. 445 446 Redfish Purge Event Log 447 448 ${status}= Run Keyword And Return Status Logging Test Binary Exist 449 Run Keyword If ${status} == ${False} Install Tarball 450 451 452Test Teardown Execution 453 [Documentation] Do the post test teardown. 454 455 FFDC On Test Case Fail 456 Redfish.Login 457 Redfish Purge Event Log 458 459 460Get Redfish Error Entries 461 [Documentation] Return Redfish error ids list. 462 ${error_uris}= redfish_utils.get_member_list /redfish/v1/Systems/system/LogServices/EventLog/Entries 463 ${error_ids}= Create List 464 465 FOR ${error_uri} IN @{error_uris} 466 ${error_id}= Fetch From Right ${error_uri} / 467 Append To List ${error_ids} ${error_id} 468 END 469 470 [Return] ${error_ids} 471 472 473Event Log Should Not Exist 474 [Documentation] Event log entries should not exist. 475 476 ${elogs}= Get Event Logs 477 Should Be Empty ${elogs} msg=System event log entry is not empty. 478 479 480Event Log Should Exist 481 [Documentation] Event log entries should exist. 482 483 ${elogs}= Get Event Logs 484 Should Not Be Empty ${elogs} msg=System event log entry is not empty. 485 486 487Verify Watchdog EventLog Content 488 [Documentation] Verify watchdog event log content. 489 490 # Example: 491 # { 492 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 493 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/31", 494 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 495 # "Created": "2019-05-31T18:41:33+00:00", 496 # "EntryType": "Event", 497 # "Id": "31", 498 # "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut", 499 # "Name": "System DBus Event Log Entry", 500 # "Severity": "Critical" 501 # } 502 503 ${elog}= Get Event Logs 504 Should Be Equal As Strings 505 ... ${elog[0]["Message"]} org.open_power.Host.Boot.Error.WatchdogTimedOut 506 ... msg=Watchdog timeout event log was not found. 507 Should Be Equal As Strings 508 ... ${elog[0]["Severity"]} Critical 509 ... msg=Watchdog timeout severity unexpected value. 510