1*** Settings *** 2Documentation Inventory of hardware resources under systems. 3 4Resource ../../../lib/bmc_redfish_resource.robot 5Resource ../../../lib/bmc_redfish_utils.robot 6Resource ../../../lib/logging_utils.robot 7Resource ../../../lib/openbmc_ffdc.robot 8Resource ../../../lib/ipmi_client.robot 9Library ../../../lib/logging_utils.py 10 11Suite Setup Suite Setup Execution 12Suite Teardown Suite Teardown Execution 13Test Setup Test Setup Execution 14Test Teardown Test Teardown Execution 15 16** Variables *** 17 18${max_num_event_logs} ${200} 19 20*** Test Cases *** 21 22Event Log Check After BMC Reboot 23 [Documentation] Check event log after BMC rebooted. 24 [Tags] Event_Log_Check_After_BMC_Reboot 25 26 Redfish Purge Event Log 27 Event Log Should Not Exist 28 29 Redfish OBMC Reboot (off) 30 31 Redfish.Login 32 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 33 34 Event Log Should Not Exist 35 36 37Event Log Check After Host Poweron 38 [Documentation] Check event log after host has booted. 39 [Tags] Event_Log_Check_After_Host_Poweron 40 41 Redfish Purge Event Log 42 Event Log Should Not Exist 43 44 Redfish Power On 45 46 Redfish.Login 47 Event Log Should Not Exist 48 49 50Create Test Event Log And Verify 51 [Documentation] Create event logs and verify via redfish. 52 [Tags] Create_Test_Event_Log_And_Verify 53 54 Create Test Error Log 55 Event Log Should Exist 56 57 58Test Event Log Persistency On Restart 59 [Documentation] Restart logging service and verify event logs. 60 [Tags] Test_Event_Log_Persistency_On_Restart 61 62 Create Test Error Log 63 Event Log Should Exist 64 65 BMC Execute Command 66 ... systemctl restart xyz.openbmc_project.Logging.service 67 Sleep 10s reason=Wait for logging service to restart properly. 68 69 Event Log Should Exist 70 71 72Test Event Entry Numbering Reset On Restart 73 [Documentation] Restart logging service and verify event logs entry starts 74 ... from entry "Id" 1. 75 [Tags] Test_Event_Entry_Numbering_Reset_On_Restart 76 77 #{ 78 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 79 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 80 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 81 # "Description": "Collection of System Event Log Entries", 82 # "Members": [ 83 # { 84 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 85 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 86 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 87 # "Created": "2019-05-29T13:19:27+00:00", 88 # "EntryType": "Event", 89 # "Id": "1", <----- Event log ID 90 # "Message": "org.open_power.Host.Error.Event", 91 # "Name": "System DBus Event Log Entry", 92 # "Severity": "Critical" 93 # } 94 # ], 95 # "Members@odata.count": 1, 96 # "Name": "System Event Log Entries" 97 #} 98 99 Create Test Error Log 100 Create Test Error Log 101 Event Log Should Exist 102 103 Redfish Purge Event Log 104 Event Log Should Not Exist 105 106 BMC Execute Command 107 ... systemctl restart xyz.openbmc_project.Logging.service 108 Sleep 10s reason=Wait for logging service to restart properly. 109 110 Create Test Error Log 111 ${elogs}= Get Event Logs 112 Should Be Equal ${elogs[0]["Id"]} 1 msg=Event log entry is not 1. 113 114 115Test Event Log Persistency On Reboot 116 [Documentation] Reboot BMC and verify event log. 117 [Tags] Test_Event_Log_Persistency_On_Reboot 118 119 Redfish Purge Event Log 120 Create Test Error Log 121 Event Log Should Exist 122 123 Redfish OBMC Reboot (off) 124 125 Redfish.Login 126 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 127 128 Event Log Should Exist 129 130 131# TODO: openbmc/openbmc-test-automation#1789 132Create Test Event Log And Verify Resolved Field 133 [Documentation] Create event log and verify "Resolved" field is 0. 134 [Tags] Create_Test_Event_Log_And_Verify_Resolved_Field 135 136 # Example Error log: 137 # "/xyz/openbmc_project/logging/entry/1": { 138 # "AdditionalData": [ 139 # "STRING=FOO" 140 # ], 141 # "Id": 1, 142 # "Message": "example.xyz.openbmc_project.Example.Elog.AutoTestSimple", 143 # "Resolved": 0, 144 # "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error", 145 # "Timestamp": 1490817164983, 146 # "associations": [] 147 # }, 148 149 # To mark an error as resolved, without deleting the error, user would 150 # set this bool property. 151 # In this test context we are making sure "Resolved" field is "0" 152 # by default. 153 154 Redfish Purge Event Log 155 Create Test Error Log 156 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 157 ${resolved}= Read Attribute ${elog_entry[0]} Resolved 158 Should Be True ${resolved} == 0 159 160 161Create Test Event Log And Verify Time Stamp 162 [Documentation] Create event logs and verify time stamp. 163 [Tags] Create_Test_Event_Log_And_Verify_Time_Stamp 164 165 #{ 166 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 167 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 168 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 169 # "Description": "Collection of System Event Log Entries", 170 # "Members": [ 171 # { 172 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 173 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 174 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 175 # "Created": "2019-05-29T13:19:27+00:00", <--- Time stamp 176 # "EntryType": "Event", 177 # "Id": "1", 178 # "Message": "org.open_power.Host.Error.Event", 179 # "Name": "System DBus Event Log Entry", 180 # "Severity": "Critical" 181 # } 182 # ], 183 # "Members@odata.count": 1, 184 # "Name": "System Event Log Entries" 185 #} 186 187 Redfish Purge Event Log 188 189 Create Test Error Log 190 Sleep 2s 191 Create Test Error Log 192 193 ${elog_entry}= Get Event Logs 194 195 # The event log generated is associated with the epoc time and unique 196 # for every error and in increasing time stamp. 197 ${time_stamp1}= Convert Date ${elog_entry[0]["Created"]} epoch 198 ${time_stamp2}= Convert Date ${elog_entry[1]["Created"]} epoch 199 200 Should Be True ${time_stamp2} > ${time_stamp1} 201 202 203Verify IPMI SEL Delete 204 [Documentation] Verify IPMI SEL delete operation. 205 [Tags] Verify_IPMI_SEL_Delete 206 207 Redfish Purge Event Log 208 Create Test Error Log 209 210 ${sel_list}= Run IPMI Standard Command sel list 211 Should Not Be Equal As Strings ${sel_list} SEL has no entries 212 213 # Example of SEL List: 214 # 4 | 04/21/2017 | 10:51:16 | System Event #0x01 | Undetermined system hardware failure | Asserted 215 216 ${sel_entry}= Fetch from Left ${sel_list} | 217 ${sel_entry}= Evaluate $sel_entry.replace(' ','') 218 ${sel_entry}= Convert To Integer 0x${sel_entry} 219 220 ${sel_delete}= Run IPMI Standard Command sel delete ${sel_entry} 221 Should Be Equal As Strings ${sel_delete} Deleted entry ${sel_entry} 222 ... case_insensitive=True 223 224 ${sel_list}= Run IPMI Standard Command sel list 225 Should Be Equal As Strings ${sel_list} SEL has no entries 226 ... case_insensitive=True 227 228 229Delete Non Existing SEL Event Entry 230 [Documentation] Delete non existing SEL event entry. 231 [Tags] Delete_Non_Existing_SEL_Event_Entry 232 233 ${sel_delete}= Run Keyword And Expect Error * 234 ... Run IPMI Standard Command sel delete 100 235 Should Contain ${sel_delete} Unable to delete entry 236 ... case_insensitive=True 237 238 239Delete Invalid SEL Event Entry 240 [Documentation] Delete invalid SEL event entry. 241 [Tags] Delete_Invalid_SEL_Event_Entry 242 243 ${sel_delete}= Run Keyword And Expect Error * 244 ... Run IPMI Standard Command sel delete abc 245 Should Contain ${sel_delete} Given SEL ID 'abc' is invalid 246 ... case_insensitive=True 247 248 249Verify IPMI SEL Event Entries 250 [Documentation] Verify IPMI SEL's entries info. 251 [Tags] Verify_IPMI_SEL_Event_Entries 252 253 # Generate error logs of random count. 254 ${count}= Evaluate random.randint(1, 5) modules=random 255 Repeat Keyword ${count} Create Test Error Log 256 257 ${sel_entries_count}= Get IPMI SEL Setting Entries 258 Should Be Equal As Strings ${sel_entries_count} ${count} 259 260 261Verify IPMI SEL Event Last Add Time 262 [Documentation] Verify IPMI SEL's last added timestamp. 263 [Tags] Verify_IPMI_SEL_Event_Last_Add_Time 264 265 Create Test Error Log 266 ${sel_time}= Run IPMI Standard Command sel time get 267 ${sel_time}= Convert Date ${sel_time} 268 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 269 270 ${sel_last_add_time}= Get IPMI SEL Setting Last Add Time 271 ${sel_last_add_time}= Convert Date ${sel_last_add_time} 272 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 273 274 ${time_diff}= 275 ... Subtract Date From Date ${sel_last_add_time} ${sel_time} 276 277 # Verify if the delay in current time check and last add SEL time 278 # is less or equals to 2 seconds. 279 Should Be True ${time_diff} <= 2 280 281 282Create Test Event Log And Delete 283 [Documentation] Create an event log and delete it. 284 [Tags] Create_Test_Event_Log_And_Delete 285 286 Create Test Error Log 287 Redfish Purge Event Log 288 Event Log Should Not Exist 289 290 291Create Multiple Test Event Logs And Delete All 292 [Documentation] Create multiple event logs and delete all. 293 [Tags] Create_Multiple_Test_Event_Logs_And_Delete_All 294 295 Create Test Error Log 296 Create Test Error Log 297 Create Test Error Log 298 Redfish Purge Event Log 299 Event Log Should Not Exist 300 301 302# TODO: openbmc/openbmc-test-automation#1789 303Create Two Test Event Logs And Delete One 304 [Documentation] Create two event logs and delete the first entry. 305 [Tags] Create_Two_Test_Eevent_Logs_And_Delete_One 306 307 Redfish Purge Event Log 308 Create Test Error Log 309 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 310 Create Test Error Log 311 Delete Error log Entry ${elog_entry[0]} 312 ${resp}= OpenBMC Get Request ${elog_entry[0]} 313 Should Be Equal As Strings ${resp.status_code} ${HTTP_NOT_FOUND} 314 315 316Verify Watchdog Timedout Event 317 [Documentation] Trigger watchdog timed out and verify event log generated. 318 [Tags] Verify_Watchdog_Timedout_Event 319 320 Redfish Power On 321 322 # Clear errors if there are any. 323 Redfish.Login 324 Redfish Purge Event Log 325 326 Trigger Host Watchdog Error 327 328 # Logging takes time to generate the timeout error. 329 Wait Until Keyword Succeeds 2 min 30 sec 330 ... Verify Watchdog EventLog Content 331 332 Redfish Power Off 333 334 335Verify Event Logs Capping 336 [Documentation] Verify event logs capping. 337 [Tags] Verify_Event_Logs_Capping 338 339 Redfish Purge Event Log 340 341 ${cmd}= Catenate for i in {1..201}; do /tmp/tarball/bin/logging-test -c 342 ... AutoTestSimple;sleep 1;done 343 BMC Execute Command ${cmd} 344 345 ${elogs}= Get Event Logs 346 ${count}= Get Length ${elogs} 347 Run Keyword If ${count} > 200 348 ... Fail Error logs created exceeded max capacity 200. 349 350 351Test Event Log Wrapping 352 [Documentation] Verify event log entries wraps when 200 max cap is reached. 353 [Tags] Test_Event_Log_Wrapping 354 355 # Restarting logging service in order to clear logs and get the next log 356 # ID set to 1. 357 BMC Execute Command 358 ... systemctl restart xyz.openbmc_project.Logging.service 359 Sleep 10s reason=Wait for logging service to restart properly. 360 361 # Create ${max_num_event_logs} event logs. 362 ${cmd}= Catenate for i in {1..${max_num_event_logs}}; do /tmp/tarball/bin/logging-test -c 363 ... AutoTestSimple;sleep 1;done 364 BMC Execute Command ${cmd} 365 366 # Verify that event logs with IDs 1 and ${max_num_event_logs} exist. 367 ${event_log}= Get Event Logs 368 369 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 370 Rprint Vars log_entries 371 Should Be Equal As Strings ${log_entries[0]["Id"]} 1 372 373 ${log_entries}= Filter Struct ${event_log} [('Id', '${max_num_event_logs}')] 374 Rprint Vars log_entries 375 Should Be Equal As Strings ${log_entries[0]["Id"]} ${max_num_event_logs} 376 377 # Create event log and verify the entry ID, ${max_num_event_logs + 1}. 378 ${next_event_log_id}= Set Variable ${max_num_event_logs + 1} 379 380 Create Test Error Log 381 382 ${event_log}= Get Event Logs 383 384 ${log_entries}= Filter Struct ${event_log} [('Id', '${next_event_log_id}')] 385 Rprint Vars log_entries 386 Should Be Equal As Strings ${log_entries[0]["Id"]} ${next_event_log_id} 387 388 # Event log 1 should be wrapped. 389 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 390 Rprint Vars log_entries 391 392 ${length_log_entries} Get Length ${log_entries} 393 Should Be Equal As Integers ${length_log_entries} 0 394 ... msg=The event log should have wrapped such that entry ID 1 is now purged. 395 396 397*** Keywords *** 398 399Suite Setup Execution 400 [Documentation] Do test case setup tasks. 401 402 Redfish.Login 403 404 Redfish Purge Event Log 405 406 ${status}= Run Keyword And Return Status Logging Test Binary Exist 407 Run Keyword If ${status} == ${False} Install Tarball 408 409 410Suite Teardown Execution 411 [Documentation] Do the post suite teardown. 412 413 Redfish.Logout 414 415 416Test Setup Execution 417 [Documentation] Do test case setup tasks. 418 419 Redfish Purge Event Log 420 421 ${status}= Run Keyword And Return Status Logging Test Binary Exist 422 Run Keyword If ${status} == ${False} Install Tarball 423 424 425Test Teardown Execution 426 [Documentation] Do the post test teardown. 427 428 FFDC On Test Case Fail 429 Redfish.Login 430 Redfish Purge Event Log 431 432 433Event Log Should Not Exist 434 [Documentation] Event log entries should not exist. 435 436 ${elogs}= Get Event Logs 437 Should Be Empty ${elogs} msg=System event log entry is not empty. 438 439 440Event Log Should Exist 441 [Documentation] Event log entries should exist. 442 443 ${elogs}= Get Event Logs 444 Should Not Be Empty ${elogs} msg=System event log entry is not empty. 445 446 447Verify Watchdog EventLog Content 448 [Documentation] Verify watchdog event log content. 449 450 # Example: 451 # { 452 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 453 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/31", 454 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 455 # "Created": "2019-05-31T18:41:33+00:00", 456 # "EntryType": "Event", 457 # "Id": "31", 458 # "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut", 459 # "Name": "System DBus Event Log Entry", 460 # "Severity": "Critical" 461 # } 462 463 ${elog}= Get Event Logs 464 Should Be Equal As Strings 465 ... ${elog[0]["Message"]} org.open_power.Host.Boot.Error.WatchdogTimedOut 466 ... msg=Watchdog timeout event log was not found. 467 Should Be Equal As Strings 468 ... ${elog[0]["Severity"]} Critical 469 ... msg=Watchdog timeout severity unexpected value. 470