1*** Settings *** 2Documentation Inventory of hardware resources under systems. 3 4Resource ../../../lib/bmc_redfish_resource.robot 5Resource ../../../lib/bmc_redfish_utils.robot 6Resource ../../../lib/logging_utils.robot 7Resource ../../../lib/openbmc_ffdc.robot 8Resource ../../../lib/ipmi_client.robot 9Library ../../../lib/logging_utils.py 10 11Suite Setup Suite Setup Execution 12Suite Teardown Suite Teardown Execution 13Test Setup Test Setup Execution 14Test Teardown Test Teardown Execution 15 16** Variables *** 17 18${max_num_event_logs} ${200} 19 20*** Test Cases *** 21 22Event Log Check After BMC Reboot 23 [Documentation] Check event log after BMC rebooted. 24 [Tags] Event_Log_Check_After_BMC_Reboot 25 26 Redfish Purge Event Log 27 Event Log Should Not Exist 28 29 Redfish OBMC Reboot (off) 30 31 Redfish.Login 32 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 33 34 Event Log Should Not Exist 35 36 37Event Log Check After Host Poweron 38 [Documentation] Check event log after host has booted. 39 [Tags] Event_Log_Check_After_Host_Poweron 40 41 Redfish Purge Event Log 42 Event Log Should Not Exist 43 44 Redfish Power On 45 46 Redfish.Login 47 Event Log Should Not Exist 48 49 50Create Test Event Log And Verify 51 [Documentation] Create event logs and verify via redfish. 52 [Tags] Create_Test_Event_Log_And_Verify 53 54 Create Test Error Log 55 Event Log Should Exist 56 57 58Delete Redfish Event Log And Verify 59 [Documentation] Delete Redfish event log and verify via Redfish. 60 [Tags] Delete_Redfish_Event_Log_And_Verify 61 62 Redfish.Login 63 Redfish Purge Event Log 64 Create Test PEL Log 65 ${elog_entry}= Get Event Logs 66 67 Redfish.Delete /redfish/v1/Systems/system/LogServices/EventLog/Entries/${elog_entry[0]["Id"]} 68 69 ${error_entries}= Get Redfish Error Entries 70 Should Be Empty ${error_entries} 71 72 73Test Event Log Persistency On Restart 74 [Documentation] Restart logging service and verify event logs. 75 [Tags] Test_Event_Log_Persistency_On_Restart 76 77 Create Test Error Log 78 Event Log Should Exist 79 80 BMC Execute Command 81 ... systemctl restart xyz.openbmc_project.Logging.service 82 Sleep 10s reason=Wait for logging service to restart properly. 83 84 Event Log Should Exist 85 86 87Test Event Entry Numbering Reset On Restart 88 [Documentation] Restart logging service and verify event logs entry starts 89 ... from entry "Id" 1. 90 [Tags] Test_Event_Entry_Numbering_Reset_On_Restart 91 92 #{ 93 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 94 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 95 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 96 # "Description": "Collection of System Event Log Entries", 97 # "Members": [ 98 # { 99 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 100 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 101 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 102 # "Created": "2019-05-29T13:19:27+00:00", 103 # "EntryType": "Event", 104 # "Id": "1", <----- Event log ID 105 # "Message": "org.open_power.Host.Error.Event", 106 # "Name": "System DBus Event Log Entry", 107 # "Severity": "Critical" 108 # } 109 # ], 110 # "Members@odata.count": 1, 111 # "Name": "System Event Log Entries" 112 #} 113 114 Create Test Error Log 115 Create Test Error Log 116 Event Log Should Exist 117 118 Redfish Purge Event Log 119 Event Log Should Not Exist 120 121 BMC Execute Command 122 ... systemctl restart xyz.openbmc_project.Logging.service 123 Sleep 10s reason=Wait for logging service to restart properly. 124 125 Create Test Error Log 126 ${elogs}= Get Event Logs 127 Should Be Equal ${elogs[0]["Id"]} 1 msg=Event log entry is not 1. 128 129 130Test Event Log Persistency On Reboot 131 [Documentation] Reboot BMC and verify event log. 132 [Tags] Test_Event_Log_Persistency_On_Reboot 133 134 Redfish Purge Event Log 135 Create Test Error Log 136 Event Log Should Exist 137 138 Redfish OBMC Reboot (off) 139 140 Redfish.Login 141 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 142 143 Event Log Should Exist 144 145 146# TODO: openbmc/openbmc-test-automation#1789 147Create Test Event Log And Verify Resolved Field 148 [Documentation] Create event log and verify "Resolved" field is 0. 149 [Tags] Create_Test_Event_Log_And_Verify_Resolved_Field 150 151 # Example Error log: 152 # "/xyz/openbmc_project/logging/entry/1": { 153 # "AdditionalData": [ 154 # "STRING=FOO" 155 # ], 156 # "Id": 1, 157 # "Message": "example.xyz.openbmc_project.Example.Elog.AutoTestSimple", 158 # "Resolved": 0, 159 # "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error", 160 # "Timestamp": 1490817164983, 161 # "Associations": [] 162 # }, 163 164 # To mark an error as resolved, without deleting the error, user would 165 # set this bool property. 166 # In this test context we are making sure "Resolved" field is "0" 167 # by default. 168 169 Redfish Purge Event Log 170 Create Test Error Log 171 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 172 ${resolved}= Read Attribute ${elog_entry[0]} Resolved 173 Should Be True ${resolved} == 0 174 175 176Create Test Event Log And Verify Time Stamp 177 [Documentation] Create event logs and verify time stamp. 178 [Tags] Create_Test_Event_Log_And_Verify_Time_Stamp 179 180 #{ 181 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 182 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 183 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 184 # "Description": "Collection of System Event Log Entries", 185 # "Members": [ 186 # { 187 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 188 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 189 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 190 # "Created": "2019-05-29T13:19:27+00:00", <--- Time stamp 191 # "EntryType": "Event", 192 # "Id": "1", 193 # "Message": "org.open_power.Host.Error.Event", 194 # "Name": "System DBus Event Log Entry", 195 # "Severity": "Critical" 196 # } 197 # ], 198 # "Members@odata.count": 1, 199 # "Name": "System Event Log Entries" 200 #} 201 202 Redfish Purge Event Log 203 204 Create Test Error Log 205 Sleep 2s 206 Create Test Error Log 207 208 ${elog_entry}= Get Event Logs 209 210 # The event log generated is associated with the epoc time and unique 211 # for every error and in increasing time stamp. 212 ${time_stamp1}= Convert Date ${elog_entry[0]["Created"]} epoch 213 ${time_stamp2}= Convert Date ${elog_entry[1]["Created"]} epoch 214 215 Should Be True ${time_stamp2} > ${time_stamp1} 216 217 218Verify IPMI SEL Delete 219 [Documentation] Verify IPMI SEL delete operation. 220 [Tags] Verify_IPMI_SEL_Delete 221 222 Redfish Purge Event Log 223 Create Test Error Log 224 225 ${sel_list}= Run IPMI Standard Command sel list 226 Should Not Be Equal As Strings ${sel_list} SEL has no entries 227 228 # Example of SEL List: 229 # 4 | 04/21/2017 | 10:51:16 | System Event #0x01 | Undetermined system hardware failure | Asserted 230 231 ${sel_entry}= Fetch from Left ${sel_list} | 232 ${sel_entry}= Evaluate $sel_entry.replace(' ','') 233 ${sel_entry}= Convert To Integer 0x${sel_entry} 234 235 ${sel_delete}= Run IPMI Standard Command sel delete ${sel_entry} 236 Should Be Equal As Strings ${sel_delete} Deleted entry ${sel_entry} 237 ... case_insensitive=True 238 239 ${sel_list}= Run IPMI Standard Command sel list 240 Should Be Equal As Strings ${sel_list} SEL has no entries 241 ... case_insensitive=True 242 243 244Delete Non Existing SEL Event Entry 245 [Documentation] Delete non existing SEL event entry. 246 [Tags] Delete_Non_Existing_SEL_Event_Entry 247 248 ${sel_delete}= Run Keyword And Expect Error * 249 ... Run IPMI Standard Command sel delete 100 250 Should Contain ${sel_delete} Unable to delete entry 251 ... case_insensitive=True 252 253 254Delete Invalid SEL Event Entry 255 [Documentation] Delete invalid SEL event entry. 256 [Tags] Delete_Invalid_SEL_Event_Entry 257 258 ${sel_delete}= Run Keyword And Expect Error * 259 ... Run IPMI Standard Command sel delete abc 260 Should Contain ${sel_delete} Given SEL ID 'abc' is invalid 261 ... case_insensitive=True 262 263 264Verify IPMI SEL Event Entries 265 [Documentation] Verify IPMI SEL's entries info. 266 [Tags] Verify_IPMI_SEL_Event_Entries 267 268 # Generate error logs of random count. 269 ${count}= Evaluate random.randint(1, 5) modules=random 270 Repeat Keyword ${count} Create Test Error Log 271 272 ${sel_entries_count}= Get IPMI SEL Setting Entries 273 Should Be Equal As Strings ${sel_entries_count} ${count} 274 275 276Verify IPMI SEL Event Last Add Time 277 [Documentation] Verify IPMI SEL's last added timestamp. 278 [Tags] Verify_IPMI_SEL_Event_Last_Add_Time 279 280 Create Test Error Log 281 ${sel_time}= Run IPMI Standard Command sel time get 282 ${sel_time}= Convert Date ${sel_time} 283 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 284 285 ${sel_last_add_time}= Get IPMI SEL Setting Last Add Time 286 ${sel_last_add_time}= Convert Date ${sel_last_add_time} 287 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 288 289 ${time_diff}= 290 ... Subtract Date From Date ${sel_last_add_time} ${sel_time} 291 292 # Verify if the delay in current time check and last add SEL time 293 # is less or equals to 2 seconds. 294 Should Be True ${time_diff} <= 2 295 296 297Create Test Event Log And Delete 298 [Documentation] Create an event log and delete it. 299 [Tags] Create_Test_Event_Log_And_Delete 300 301 Create Test Error Log 302 Redfish Purge Event Log 303 Event Log Should Not Exist 304 305 306Create Multiple Test Event Logs And Delete All 307 [Documentation] Create multiple event logs and delete all. 308 [Tags] Create_Multiple_Test_Event_Logs_And_Delete_All 309 310 Create Test Error Log 311 Create Test Error Log 312 Create Test Error Log 313 Redfish Purge Event Log 314 Event Log Should Not Exist 315 316 317# TODO: openbmc/openbmc-test-automation#1789 318Create Two Test Event Logs And Delete One 319 [Documentation] Create two event logs and delete the first entry. 320 [Tags] Create_Two_Test_Event_Logs_And_Delete_One 321 322 Redfish Purge Event Log 323 Create Test Error Log 324 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 325 Create Test Error Log 326 Delete Error log Entry ${elog_entry[0]} 327 ${resp}= OpenBMC Get Request ${elog_entry[0]} 328 Should Be Equal As Strings ${resp.status_code} ${HTTP_NOT_FOUND} 329 330 331Verify Watchdog Timedout Event 332 [Documentation] Trigger watchdog timed out and verify event log generated. 333 [Tags] Verify_Watchdog_Timedout_Event 334 335 Redfish Power On 336 337 # Clear errors if there are any. 338 Redfish.Login 339 Redfish Purge Event Log 340 341 Trigger Host Watchdog Error 342 343 # Logging takes time to generate the timeout error. 344 Wait Until Keyword Succeeds 2 min 30 sec 345 ... Verify Watchdog EventLog Content 346 347 348Verify Event Logs Capping 349 [Documentation] Verify event logs capping. 350 [Tags] Verify_Event_Logs_Capping 351 352 Redfish Purge Event Log 353 354 ${cmd}= Catenate for i in {1..201}; do /tmp/tarball/bin/logging-test -c 355 ... AutoTestSimple;sleep 1;done 356 BMC Execute Command ${cmd} 357 358 ${elogs}= Get Event Logs 359 ${count}= Get Length ${elogs} 360 Run Keyword If ${count} > 200 361 ... Fail Error logs created exceeded max capacity 200. 362 363 364Test Event Log Wrapping 365 [Documentation] Verify event log entries wraps when 200 max cap is reached. 366 [Tags] Test_Event_Log_Wrapping 367 368 # Restarting logging service in order to clear logs and get the next log 369 # ID set to 1. 370 BMC Execute Command 371 ... systemctl restart xyz.openbmc_project.Logging.service 372 Sleep 10s reason=Wait for logging service to restart properly. 373 374 # Create ${max_num_event_logs} event logs. 375 ${cmd}= Catenate for i in {1..${max_num_event_logs}}; do /tmp/tarball/bin/logging-test -c 376 ... AutoTestSimple;sleep 1;done 377 BMC Execute Command ${cmd} 378 379 # Verify that event logs with IDs 1 and ${max_num_event_logs} exist. 380 ${event_log}= Get Event Logs 381 382 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 383 Rprint Vars log_entries 384 Should Be Equal As Strings ${log_entries[0]["Id"]} 1 385 386 ${log_entries}= Filter Struct ${event_log} [('Id', '${max_num_event_logs}')] 387 Rprint Vars log_entries 388 Should Be Equal As Strings ${log_entries[0]["Id"]} ${max_num_event_logs} 389 390 # Create event log and verify the entry ID, ${max_num_event_logs + 1}. 391 ${next_event_log_id}= Set Variable ${max_num_event_logs + 1} 392 393 Create Test Error Log 394 395 ${event_log}= Get Event Logs 396 397 ${log_entries}= Filter Struct ${event_log} [('Id', '${next_event_log_id}')] 398 Rprint Vars log_entries 399 Should Be Equal As Strings ${log_entries[0]["Id"]} ${next_event_log_id} 400 401 # Event log 1 should be wrapped. 402 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 403 Rprint Vars log_entries 404 405 ${length_log_entries} Get Length ${log_entries} 406 Should Be Equal As Integers ${length_log_entries} 0 407 ... msg=The event log should have wrapped such that entry ID 1 is now purged. 408 409 410*** Keywords *** 411 412Suite Setup Execution 413 [Documentation] Do test case setup tasks. 414 415 Redfish.Login 416 417 Redfish Purge Event Log 418 419 ${status}= Run Keyword And Return Status Logging Test Binary Exist 420 Run Keyword If ${status} == ${False} Install Tarball 421 422 423Suite Teardown Execution 424 [Documentation] Do the post suite teardown. 425 426 Redfish.Logout 427 428 429Test Setup Execution 430 [Documentation] Do test case setup tasks. 431 432 Redfish Purge Event Log 433 434 ${status}= Run Keyword And Return Status Logging Test Binary Exist 435 Run Keyword If ${status} == ${False} Install Tarball 436 437 438Test Teardown Execution 439 [Documentation] Do the post test teardown. 440 441 FFDC On Test Case Fail 442 Redfish.Login 443 Redfish Purge Event Log 444 445 446Get Redfish Error Entries 447 [Documentation] Return Redfish error ids list. 448 ${error_uris}= redfish_utils.get_member_list /redfish/v1/Systems/system/LogServices/EventLog/Entries 449 ${error_ids}= Create List 450 451 FOR ${error_uri} IN @{error_uris} 452 ${error_id}= Fetch From Right ${error_uri} / 453 Append To List ${error_ids} ${error_id} 454 END 455 456 [Return] ${error_ids} 457 458 459Event Log Should Not Exist 460 [Documentation] Event log entries should not exist. 461 462 ${elogs}= Get Event Logs 463 Should Be Empty ${elogs} msg=System event log entry is not empty. 464 465 466Event Log Should Exist 467 [Documentation] Event log entries should exist. 468 469 ${elogs}= Get Event Logs 470 Should Not Be Empty ${elogs} msg=System event log entry is not empty. 471 472 473Verify Watchdog EventLog Content 474 [Documentation] Verify watchdog event log content. 475 476 # Example: 477 # { 478 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 479 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/31", 480 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 481 # "Created": "2019-05-31T18:41:33+00:00", 482 # "EntryType": "Event", 483 # "Id": "31", 484 # "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut", 485 # "Name": "System DBus Event Log Entry", 486 # "Severity": "Critical" 487 # } 488 489 ${elog}= Get Event Logs 490 Should Be Equal As Strings 491 ... ${elog[0]["Message"]} org.open_power.Host.Boot.Error.WatchdogTimedOut 492 ... msg=Watchdog timeout event log was not found. 493 Should Be Equal As Strings 494 ... ${elog[0]["Severity"]} Critical 495 ... msg=Watchdog timeout severity unexpected value. 496