1*** Settings *** 2Documentation Inventory of hardware resources under systems. 3 4Resource ../../../lib/bmc_redfish_resource.robot 5Resource ../../../lib/bmc_redfish_utils.robot 6Resource ../../../lib/logging_utils.robot 7Resource ../../../lib/openbmc_ffdc.robot 8Resource ../../../lib/ipmi_client.robot 9Library ../../../lib/logging_utils.py 10 11Test Setup Test Setup Execution 12Test Teardown Test Teardown Execution 13Suite Teardown Suite Teardown Execution 14 15** Variables *** 16 17${max_num_event_logs} ${200} 18 19*** Test Cases *** 20 21Event Log Check After BMC Reboot 22 [Documentation] Check event log after BMC rebooted. 23 [Tags] Event_Log_Check_After_BMC_Reboot 24 25 Redfish Purge Event Log 26 Event Log Should Not Exist 27 28 Redfish OBMC Reboot (off) 29 30 Redfish.Login 31 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 32 33 Event Log Should Not Exist 34 35 36Event Log Check After Host Poweron 37 [Documentation] Check event log after host has booted. 38 [Tags] Event_Log_Check_After_Host_Poweron 39 40 Redfish Purge Event Log 41 Event Log Should Not Exist 42 43 Redfish Power On 44 45 Redfish.Login 46 Event Log Should Not Exist 47 48 49Create Test Event Log And Verify 50 [Documentation] Create event logs and verify via redfish. 51 [Tags] Create_Test_Event_Log_And_Verify 52 53 Create Test Error Log 54 Event Log Should Exist 55 56 57Test Event Log Persistency On Restart 58 [Documentation] Restart logging service and verify event logs. 59 [Tags] Test_Event_Log_Persistency_On_Restart 60 61 Create Test Error Log 62 Event Log Should Exist 63 64 BMC Execute Command 65 ... systemctl restart xyz.openbmc_project.Logging.service 66 Sleep 10s reason=Wait for logging service to restart properly. 67 68 Event Log Should Exist 69 70 71Test Event Entry Numbering Reset On Restart 72 [Documentation] Restart logging service and verify event logs entry starts 73 ... from entry "Id" 1. 74 [Tags] Test_Event_Entry_Numbering_Reset_On_Restart 75 76 #{ 77 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 78 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 79 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 80 # "Description": "Collection of System Event Log Entries", 81 # "Members": [ 82 # { 83 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 84 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 85 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 86 # "Created": "2019-05-29T13:19:27+00:00", 87 # "EntryType": "Event", 88 # "Id": "1", <----- Event log ID 89 # "Message": "org.open_power.Host.Error.Event", 90 # "Name": "System DBus Event Log Entry", 91 # "Severity": "Critical" 92 # } 93 # ], 94 # "Members@odata.count": 1, 95 # "Name": "System Event Log Entries" 96 #} 97 98 Create Test Error Log 99 Create Test Error Log 100 Event Log Should Exist 101 102 Redfish Purge Event Log 103 Event Log Should Not Exist 104 105 BMC Execute Command 106 ... systemctl restart xyz.openbmc_project.Logging.service 107 Sleep 10s reason=Wait for logging service to restart properly. 108 109 Create Test Error Log 110 ${elogs}= Get Event Logs 111 Should Be Equal ${elogs[0]["Id"]} 1 msg=Event log entry is not 1. 112 113 114Test Event Log Persistency On Reboot 115 [Documentation] Reboot BMC and verify event log. 116 [Tags] Test_Event_Log_Persistency_On_Reboot 117 118 Redfish Purge Event Log 119 Create Test Error Log 120 Event Log Should Exist 121 122 Redfish OBMC Reboot (off) 123 124 Redfish.Login 125 Wait Until Keyword Succeeds 1 mins 15 secs Redfish.Get ${EVENT_LOG_URI}Entries 126 127 Event Log Should Exist 128 129 130# TODO: openbmc/openbmc-test-automation#1789 131Create Test Event Log And Verify Resolved Field 132 [Documentation] Create event log and verify "Resolved" field is 0. 133 [Tags] Create_Test_Event_Log_And_Verify_Resolved_Field 134 135 # Example Error log: 136 # "/xyz/openbmc_project/logging/entry/1": { 137 # "AdditionalData": [ 138 # "STRING=FOO" 139 # ], 140 # "Id": 1, 141 # "Message": "example.xyz.openbmc_project.Example.Elog.AutoTestSimple", 142 # "Resolved": 0, 143 # "Severity": "xyz.openbmc_project.Logging.Entry.Level.Error", 144 # "Timestamp": 1490817164983, 145 # "associations": [] 146 # }, 147 148 # To mark an error as resolved, without deleting the error, user would 149 # set this bool property. 150 # In this test context we are making sure "Resolved" field is "0" 151 # by default. 152 153 Redfish Purge Event Log 154 Create Test Error Log 155 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 156 ${resolved}= Read Attribute ${elog_entry[0]} Resolved 157 Should Be True ${resolved} == 0 158 159 160Create Test Event Log And Verify Time Stamp 161 [Documentation] Create event logs and verify time stamp. 162 [Tags] Create_Test_Event_Log_And_Verify_Time_Stamp 163 164 #{ 165 # "@odata.context": "/redfish/v1/$metadata#LogEntryCollection.LogEntryCollection", 166 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries", 167 # "@odata.type": "#LogEntryCollection.LogEntryCollection", 168 # "Description": "Collection of System Event Log Entries", 169 # "Members": [ 170 # { 171 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 172 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/1", 173 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 174 # "Created": "2019-05-29T13:19:27+00:00", <--- Time stamp 175 # "EntryType": "Event", 176 # "Id": "1", 177 # "Message": "org.open_power.Host.Error.Event", 178 # "Name": "System DBus Event Log Entry", 179 # "Severity": "Critical" 180 # } 181 # ], 182 # "Members@odata.count": 1, 183 # "Name": "System Event Log Entries" 184 #} 185 186 Redfish Purge Event Log 187 188 Create Test Error Log 189 Sleep 2s 190 Create Test Error Log 191 192 ${elog_entry}= Get Event Logs 193 194 # The event log generated is associated with the epoc time and unique 195 # for every error and in increasing time stamp. 196 ${time_stamp1}= Convert Date ${elog_entry[0]["Created"]} epoch 197 ${time_stamp2}= Convert Date ${elog_entry[1]["Created"]} epoch 198 199 Should Be True ${time_stamp2} > ${time_stamp1} 200 201 202Verify IPMI SEL Delete 203 [Documentation] Verify IPMI SEL delete operation. 204 [Tags] Verify_IPMI_SEL_Delete 205 206 Redfish Purge Event Log 207 Create Test Error Log 208 209 ${sel_list}= Run IPMI Standard Command sel list 210 Should Not Be Equal As Strings ${sel_list} SEL has no entries 211 212 # Example of SEL List: 213 # 4 | 04/21/2017 | 10:51:16 | System Event #0x01 | Undetermined system hardware failure | Asserted 214 215 ${sel_entry}= Fetch from Left ${sel_list} | 216 ${sel_entry}= Evaluate $sel_entry.replace(' ','') 217 ${sel_entry}= Convert To Integer 0x${sel_entry} 218 219 ${sel_delete}= Run IPMI Standard Command sel delete ${sel_entry} 220 Should Be Equal As Strings ${sel_delete} Deleted entry ${sel_entry} 221 ... case_insensitive=True 222 223 ${sel_list}= Run IPMI Standard Command sel list 224 Should Be Equal As Strings ${sel_list} SEL has no entries 225 ... case_insensitive=True 226 227 228Delete Non Existing SEL Event Entry 229 [Documentation] Delete non existing SEL event entry. 230 [Tags] Delete_Non_Existing_SEL_Event_Entry 231 232 ${sel_delete}= Run Keyword And Expect Error * 233 ... Run IPMI Standard Command sel delete 100 234 Should Contain ${sel_delete} Unable to delete entry 235 ... case_insensitive=True 236 237 238Delete Invalid SEL Event Entry 239 [Documentation] Delete invalid SEL event entry. 240 [Tags] Delete_Invalid_SEL_Event_Entry 241 242 ${sel_delete}= Run Keyword And Expect Error * 243 ... Run IPMI Standard Command sel delete abc 244 Should Contain ${sel_delete} Given SEL ID 'abc' is invalid 245 ... case_insensitive=True 246 247 248Verify IPMI SEL Event Entries 249 [Documentation] Verify IPMI SEL's entries info. 250 [Tags] Verify_IPMI_SEL_Event_Entries 251 252 # Generate error logs of random count. 253 ${count}= Evaluate random.randint(1, 5) modules=random 254 Repeat Keyword ${count} Create Test Error Log 255 256 ${sel_entries_count}= Get IPMI SEL Setting Entries 257 Should Be Equal As Strings ${sel_entries_count} ${count} 258 259 260Verify IPMI SEL Event Last Add Time 261 [Documentation] Verify IPMI SEL's last added timestamp. 262 [Tags] Verify_IPMI_SEL_Event_Last_Add_Time 263 264 Create Test Error Log 265 ${sel_time}= Run IPMI Standard Command sel time get 266 ${sel_time}= Convert Date ${sel_time} 267 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 268 269 ${sel_last_add_time}= Get IPMI SEL Setting Last Add Time 270 ${sel_last_add_time}= Convert Date ${sel_last_add_time} 271 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=True 272 273 ${time_diff}= 274 ... Subtract Date From Date ${sel_last_add_time} ${sel_time} 275 276 # Verify if the delay in current time check and last add SEL time 277 # is less or equals to 2 seconds. 278 Should Be True ${time_diff} <= 2 279 280 281Create Test Event Log And Delete 282 [Documentation] Create an event log and delete it. 283 [Tags] Create_Test_Event_Log_And_Delete 284 285 Create Test Error Log 286 Redfish Purge Event Log 287 Event Log Should Not Exist 288 289 290Create Multiple Test Event Logs And Delete All 291 [Documentation] Create multiple event logs and delete all. 292 [Tags] Create_Multiple_Test_Event_Logs_And_Delete_All 293 294 Create Test Error Log 295 Create Test Error Log 296 Create Test Error Log 297 Redfish Purge Event Log 298 Event Log Should Not Exist 299 300 301# TODO: openbmc/openbmc-test-automation#1789 302Create Two Test Event Logs And Delete One 303 [Documentation] Create two event logs and delete the first entry. 304 [Tags] Create_Two_Test_Eevent_Logs_And_Delete_One 305 306 Redfish Purge Event Log 307 Create Test Error Log 308 ${elog_entry}= Get URL List ${BMC_LOGGING_ENTRY} 309 Create Test Error Log 310 Delete Error log Entry ${elog_entry[0]} 311 ${resp}= OpenBMC Get Request ${elog_entry[0]} 312 Should Be Equal As Strings ${resp.status_code} ${HTTP_NOT_FOUND} 313 314 315Verify Watchdog Timedout Event 316 [Documentation] Trigger watchdog timed out and verify event log generated. 317 [Tags] Verify_Watchdog_Timedout_Event 318 319 Redfish Power On 320 321 # Clear errors if there are any. 322 Redfish.Login 323 Redfish Purge Event Log 324 325 Trigger Host Watchdog Error 326 327 # Logging takes time to generate the timeout error. 328 Wait Until Keyword Succeeds 2 min 30 sec 329 ... Verify Watchdog EventLog Content 330 331 Redfish Power Off 332 333 334Verify Event Logs Capping 335 [Documentation] Verify event logs capping. 336 [Tags] Verify_Event_Logs_Capping 337 338 Redfish Purge Event Log 339 340 ${cmd}= Catenate for i in {1..201}; do /tmp/tarball/bin/logging-test -c 341 ... AutoTestSimple;sleep 1;done 342 BMC Execute Command ${cmd} 343 344 ${elogs}= Get Event Logs 345 ${count}= Get Length ${elogs} 346 Run Keyword If ${count} > 200 347 ... Fail Error logs created exceeded max capacity 200. 348 349 350Test Event Log Wrapping 351 [Documentation] Verify event log entries wraps when 200 max cap is reached. 352 [Tags] Test_Event_Log_Wrapping 353 354 # Restarting logging service in order to clear logs and get the next log 355 # ID set to 1. 356 BMC Execute Command 357 ... systemctl restart xyz.openbmc_project.Logging.service 358 Sleep 10s reason=Wait for logging service to restart properly. 359 360 # Create ${max_num_event_logs} event logs. 361 ${cmd}= Catenate for i in {1..${max_num_event_logs}}; do /tmp/tarball/bin/logging-test -c 362 ... AutoTestSimple;sleep 1;done 363 BMC Execute Command ${cmd} 364 365 # Verify that event logs with IDs 1 and ${max_num_event_logs} exist. 366 ${event_log}= Get Event Logs 367 368 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 369 Rprint Vars log_entries fmt=terse 370 Should Be Equal As Strings ${log_entries[0]["Id"]} 1 371 372 ${log_entries}= Filter Struct ${event_log} [('Id', '${max_num_event_logs}')] 373 Rprint Vars log_entries fmt=terse 374 Should Be Equal As Strings ${log_entries[0]["Id"]} ${max_num_event_logs} 375 376 # Create event log and verify the entry ID, ${max_num_event_logs + 1}. 377 ${next_event_log_id}= Set Variable ${max_num_event_logs + 1} 378 379 Create Test Error Log 380 381 ${event_log}= Get Event Logs 382 383 ${log_entries}= Filter Struct ${event_log} [('Id', '${next_event_log_id}')] 384 Rprint Vars log_entries fmt=terse 385 Should Be Equal As Strings ${log_entries[0]["Id"]} ${next_event_log_id} 386 387 # Event log 1 should be wrapped. 388 ${log_entries}= Filter Struct ${event_log} [('Id', '1')] 389 Rprint Vars log_entries fmt=terse 390 391 ${length_log_entries} Get Length ${log_entries} 392 Should Be Equal As Integers ${length_log_entries} 0 393 ... msg=The event log should have wrapped such that entry ID 1 is now purged. 394 395 396*** Keywords *** 397 398Suite Teardown Execution 399 [Documentation] Do the post suite teardown. 400 401 Redfish.Logout 402 403 404Test Setup Execution 405 [Documentation] Do test case setup tasks. 406 407 Redfish.Login 408 409 Redfish Purge Event Log 410 411 ${status}= Run Keyword And Return Status Logging Test Binary Exist 412 Run Keyword If ${status} == ${False} Install Tarball 413 414 415Test Teardown Execution 416 [Documentation] Do the post test teardown. 417 418 FFDC On Test Case Fail 419 Redfish.Login 420 Redfish Purge Event Log 421 422 423Event Log Should Not Exist 424 [Documentation] Event log entries should not exist. 425 426 ${elogs}= Get Event Logs 427 Should Be Empty ${elogs} msg=System event log entry is not empty. 428 429 430Event Log Should Exist 431 [Documentation] Event log entries should exist. 432 433 ${elogs}= Get Event Logs 434 Should Not Be Empty ${elogs} msg=System event log entry is not empty. 435 436 437Verify Watchdog EventLog Content 438 [Documentation] Verify watchdog event log content. 439 440 # Example: 441 # { 442 # "@odata.context": "/redfish/v1/$metadata#LogEntry.LogEntry", 443 # "@odata.id": "/redfish/v1/Systems/system/LogServices/EventLog/Entries/31", 444 # "@odata.type": "#LogEntry.v1_4_0.LogEntry", 445 # "Created": "2019-05-31T18:41:33+00:00", 446 # "EntryType": "Event", 447 # "Id": "31", 448 # "Message": "org.open_power.Host.Boot.Error.WatchdogTimedOut", 449 # "Name": "System DBus Event Log Entry", 450 # "Severity": "Critical" 451 # } 452 453 ${elog}= Get Event Logs 454 Should Be Equal As Strings 455 ... ${elog[0]["Message"]} org.open_power.Host.Boot.Error.WatchdogTimedOut 456 ... msg=Watchdog timeout event log was not found. 457 Should Be Equal As Strings 458 ... ${elog[0]["Severity"]} Critical 459 ... msg=Watchdog timeout severity unexpected value. 460