1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17${CMD_INTERNAL_FAILURE} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 18... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 19... xyz.openbmc_project.Logging.Entry.Level.Error 0 20 21${CMD_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 22... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.Timeout 23... xyz.openbmc_project.Logging.Entry.Level.Error 2 "TIMEOUT_IN_MSEC" "5" 24... "CALLOUT_INVENTORY_PATH" "/xyz/openbmc_project/inventory/system/chassis/motherboard" 25 26${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 27... xyz.openbmc_project.Logging.Create Create ssa{ss} org.open_power.Logging.Error.TestError1 28... xyz.openbmc_project.Logging.Entry.Level.Error 0 29 30${CMD_INFORMATIONAL_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 31... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.TestError2 32... xyz.openbmc_project.Logging.Entry.Level.Informational 0 33 34${CMD_INVENTORY_PREFIX} busctl get-property xyz.openbmc_project.Inventory.Manager 35... /xyz/openbmc_project/inventory/system/chassis/motherboard 36 37${CMD_UNRECOVERABLE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 38... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 39... xyz.openbmc_project.Logging.Entry.Level.Error 0 40 41${CMD_PREDICTIVE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 42... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 43... xyz.openbmc_project.Logging.Entry.Level.Warning 0 44 45${CMD_UNRECOVERABLE_HOST_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 46... xyz.openbmc_project.Logging.Create Create ssa{ss} 47... xyz.openbmc_project.Host.Error.Event xyz.openbmc_project.Logging.Entry.Level.Error 1 48... RAWPEL /tmp/FILE_NBMC_UNRECOVERABLE 49 50@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 51 52*** Test Cases *** 53 54Create Test PEL Log And Verify 55 [Documentation] Create PEL log using busctl command and verify via peltool. 56 [Tags] Create_Test_PEL_Log_And_Verify 57 58 Redfish Purge Event Log 59 Create Test PEL Log 60 ${pel_id}= Get PEL Log Via BMC CLI 61 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 62 63 64Verify PEL Log Details 65 [Documentation] Verify PEL log details via peltool. 66 [Tags] Verify_PEL_Log_Details 67 68 Redfish Purge Event Log 69 70 ${bmc_time1}= CLI Get BMC DateTime 71 Create Test PEL Log 72 ${bmc_time2}= CLI Get BMC DateTime 73 74 ${pel_records}= Peltool -l 75 76 # Example output from 'Peltool -l': 77 # pel_records: 78 # [0x50000012]: 79 # [CreatorID]: BMC 80 # [CompID]: 0x1000 81 # [PLID]: 0x50000012 82 # [Subsystem]: BMC Firmware 83 # [Message]: An application had an internal failure 84 # [SRC]: BD8D1002 85 # [Commit Time]: 03/02/2020 09:35:15 86 # [Sev]: Unrecoverable Error 87 88 ${ids}= Get Dictionary Keys ${pel_records} 89 ${id}= Get From List ${ids} 0 90 91 @{pel_fields}= Create List CreatorID Subsystem Message Sev 92 FOR ${field} IN @{pel_fields} 93 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 94 END 95 96 Valid Value pel_records['${id}']['PLID'] ['${id}'] 97 98 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 99 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 100 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 101 102 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 103 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 104 105 # Convert BMC and PEL time to epoch time before comparing. 106 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 107 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 108 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 109 110 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 111 112 113Verify Mandatory Sections Of Error Log PEL 114 [Documentation] Verify mandatory sections of error log PEL. 115 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 116 117 Create Test PEL Log 118 119 ${pel_ids}= Get PEL Log Via BMC CLI 120 ${pel_id}= Get From List ${pel_ids} -1 121 ${pel_output}= Peltool -i ${pel_id} 122 ${pel_sections}= Get Dictionary Keys ${pel_output} 123 124 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 125 126 127Verify PEL Log Persistence After BMC Reboot 128 [Documentation] Verify PEL log persistence after BMC reboot. 129 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 130 131 Create Test PEL Log 132 ${pel_before_reboot}= Get PEL Log Via BMC CLI 133 134 Redfish OBMC Reboot (off) 135 ${pel_after_reboot}= Get PEL Log Via BMC CLI 136 137 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 138 139 140Verify PEL ID Numbering 141 [Documentation] Verify PEL ID numbering. 142 [Tags] Verify_PEL_ID_Numbering 143 144 Redfish Purge Event Log 145 Create Test PEL Log 146 Create Test PEL Log 147 148 ${pel_ids}= Get PEL Log Via BMC CLI 149 150 # Example of PEL IDs from PEL logs. 151 # [0x50000012]: <--- First PEL ID 152 # [CreatorID]: BMC 153 # [CompID]: 0x1000 154 # [PLID]: 0x50000012 155 # [Subsystem]: BMC Firmware 156 # [Message]: An application had an internal failure 157 # [SRC]: BD8D1002 158 # [Commit Time]: 03/02/2020 09:35:15 159 # [Sev]: Unrecoverable Error 160 # 161 # [0x50000013]: <--- Second PEL ID 162 # [CreatorID]: BMC 163 # [CompID]: 0x1000 164 # [PLID]: 0x50000013 165 # [Subsystem]: BMC Firmware 166 # [Message]: An application had an internal failure 167 # [SRC]: BD8D1002 168 # [Commit Time]: 03/02/2020 09:35:15 169 # [Sev]: Unrecoverable Error 170 171 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 172 173Verify Machine Type Model And Serial Number 174 [Documentation] Verify machine type model and serial number from PEL. 175 [Tags] Verify_Machine_Type_Model_And_Serial_Number 176 177 Create Test PEL Log 178 179 ${pel_ids}= Get PEL Log Via BMC CLI 180 ${id}= Get From List ${pel_ids} -1 181 182 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 183 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 184 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 185 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 186 187 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 188 # [Failing MTMS]: 189 # [Created by]: 0x2000 190 # [Machine Type Model]: 1234-ABC <---- Machine type 191 # [Section Version]: 1 192 # [Serial Number]: ABCDEFG <---- Serial number 193 # [Sub-section type]: 0 194 195 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 196 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 197 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 198 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 199 200 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 201 Valid Value pel_serial_number ['${redfish_serial_number}'] 202 203 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 204 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 205 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 206 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 207 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 208 209 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 210 Valid Value pel_serial_number ['${redfish_serial_number}'] 211 212 213Verify Host Off State From PEL 214 [Documentation] Verify Host off state from PEL. 215 [Tags] Verify_Host_Off_State_From_PEL 216 217 Redfish Power Off stack_mode=skip 218 Create Test PEL Log 219 220 ${pel_ids}= Get PEL Log Via BMC CLI 221 ${id}= Get From List ${pel_ids} -1 222 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 223 224 Valid Value pel_host_state ['Off'] 225 226 227Verify BMC Version From PEL 228 [Documentation] Verify BMC Version from PEL. 229 [Tags] Verify_BMC_Version_From_PEL 230 231 Create Test PEL Log 232 233 ${pel_ids}= Get PEL Log Via BMC CLI 234 ${id}= Get From List ${pel_ids} -1 235 ${pel_bmc_version}= Get PEL Field Value ${id} User Data BMC Version ID 236 237 ${bmc_version}= Get BMC Version 238 Valid Value bmc_version ['${bmc_version}'] 239 240 241Verify PEL Log After Host Poweron 242 [Documentation] Verify PEL log generation while booting host. 243 [Tags] Verify_PEL_Log_After_Host_Poweron 244 245 Redfish Power Off stack_mode=skip 246 Redfish Purge Event Log 247 Redfish Power On stack_mode=skip 248 249 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 250 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 251 252 # Get BMC created non-infomational error. 253 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 254 255 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 256 257 258Verify BMC Event Log ID 259 [Documentation] Verify BMC Event Log ID from PEL. 260 [Tags] Verify_BMC_Event_Log_ID 261 262 Redfish Purge Event Log 263 Create Test PEL Log 264 265 ${pel_ids}= Get PEL Log Via BMC CLI 266 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 267 268 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 269 # [Private Header]: 270 # [Created at]: 08/24/1928 12:04:06 271 # [Created by]: 0x584D 272 # [Sub-section type]: 0 273 # [Entry Id]: 0x50000BB7 274 # [Platform Log Id]: 0x8200061D 275 # [CSSVER]: 276 # [Section Version]: 1 277 # [Creator Subsystem]: PHYP 278 # [BMC Event Log Id]: 341 <---- BMC event log id value 279 # [Committed at]: 03/25/1920 12:06:22 280 281 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 282 283 # Example of redfish_event_logs output: 284 # redfish_event_logs: 285 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 286 # [Name]: System Event Log Entries 287 # [Members@odata.count]: 1 288 # [@odata.type]: #LogEntryCollection.LogEntryCollection 289 # [Description]: Collection of System Event Log Entries 290 # [Members]: 291 # [0]: 292 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 293 # [Name]: System Event Log Entry 294 # [Severity]: Critical 295 # [EntryType]: Event 296 # [Created]: 2020-04-02T07:25:13+00:00 297 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 298 # [Id]: 235 <----- Event log ID 299 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 300 301 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 302 303 304Verify FRU Callout 305 [Documentation] Verify FRU callout entries from PEL log. 306 [Tags] Verify_FRU_Callout 307 308 Create Test PEL Log FRU Callout 309 310 ${pel_ids}= Get PEL Log Via BMC CLI 311 ${id}= Get From List ${pel_ids} -1 312 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 313 314 # Example of PEL Callout Section from "peltool -i <id>" command. 315 # [Callouts]: 316 # [0]: 317 # [FRU Type]: Normal Hardware FRU 318 # [Priority]: Mandatory, replace all with this type as a unit 319 # [Location Code]: U78DA.ND1.1234567-P0 320 # [Part Number]: F191014 321 # [CCIN]: 2E2D 322 # [Serial Number]: YL2E2D010000 323 # [Callout Count]: 1 324 325 Valid Value pel_callout_section['Callout Count'] ['1'] 326 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 327 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 328 329 # Verify Location Code field of PEL callout with motherboard's Location Code. 330 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 331 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 332 ... ${busctl_output[0].split('"')[1].strip('"')} 333 334 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 335 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 336 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 337 338 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 339 ${busctl_output}= BMC Execute Command 340 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 341 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 342 ... ${busctl_output[0].split('"')[1].strip('"')} 343 344 345Verify Procedure And Symbolic FRU Callout 346 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 347 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 348 349 Create Test PEL Log Procedure And Symbolic FRU Callout 350 351 ${pel_ids}= Get PEL Log Via BMC CLI 352 ${id}= Get From List ${pel_ids} -1 353 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 354 355 # Example of PEL Callout Section from "peltool -i <id>" command. 356 # [Callouts]: 357 # [0]: 358 # [Priority]: Mandatory, replace all with this type as a unit 359 # [Procedure Number]: BMCSP02 360 # [FRU Type]: Maintenance Procedure Required 361 # [1]: 362 # [Priority]: Medium Priority 363 # [Part Number]: SVCDOCS 364 # [FRU Type]: Symbolic FRU 365 # [Callout Count]: 2 366 367 Valid Value pel_callout_section['Callout Count'] ['2'] 368 369 # Verify procedural callout info. 370 371 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 372 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 373 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 374 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 375 376 # Verify procedural callout info. 377 378 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 379 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 380 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 381 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 382 383 384Verify PEL Log Entry For Event Log 385 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 386 [Tags] Verify_PEL_Log_Entry_For_Event_Log 387 388 Redfish Purge Event Log 389 # Create an internal failure error log. 390 BMC Execute Command ${CMD_INTERNAL_FAILURE} 391 392 ${elog_entry}= Get Event Logs 393 # Example of Redfish event logs: 394 # elog_entry: 395 # [0]: 396 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 397 # [Created]: 2020-04-20T01:55:22+00:00 398 # [Id]: 1 399 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 400 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 401 # [EntryType]: Event 402 # [Severity]: Critical 403 # [Name]: System Event Log Entry 404 405 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 406 407 ${pel_records}= Peltool -l 408 # Example output from 'Peltool -l': 409 # pel_records: 410 # [0x50000023]: 411 # [SRC]: BD8D1002 412 # [CreatorID]: BMC 413 # [Message]: An application had an internal failure 414 # [CompID]: 0x1000 415 # [PLID]: 0x50000023 416 # [Commit Time]: 04/20/2020 01:55:22 417 # [Subsystem]: BMC Firmware 418 # [Sev]: Unrecoverable Error 419 420 ${ids}= Get Dictionary Keys ${pel_records} 421 ${id}= Get From List ${ids} 0 422 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 423 ... date_format=%m/%d/%Y %H:%M:%S 424 425 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 426 Should Contain Any ${pel_records['${id}']['Message']} internal failure ignore_case=True 427 Should Contain Any ${elog_entry[0]['Message']} InternalFailure ignore_case=True 428 429 Should Be Equal ${redfish_log_time} ${pel_log_time} 430 431 432Verify Delete All PEL 433 [Documentation] Verify deleting all PEL logs. 434 [Tags] Verify_Delete_All_PEL 435 436 Create Test PEL Log 437 Create Test PEL Log 438 Peltool --delete-all False 439 440 ${pel_ids}= Get PEL Log Via BMC CLI 441 Should Be Empty ${pel_ids} 442 443 444Verify Informational Error Log 445 [Documentation] Create an informational error log and verify. 446 [Tags] Verify_Informational_Error_Log 447 448 Redfish Purge Event Log 449 # Create an informational error log. 450 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 451 ${pel_records}= Peltool -lfh 452 453 # An example of information error log data: 454 # { 455 # "0x500006A0": { 456 # "SRC": "BD8D1002", 457 # "Message": "An application had an internal failure", 458 # "PLID": "0x500006A0", 459 # "CreatorID": "BMC", 460 # "Subsystem": "BMC Firmware", 461 # "Commit Time": "10/14/2020 11:41:38", 462 # "Sev": "Informational Event", 463 # "CompID": "0x1000" 464 # } 465 # } 466 467 ${ids}= Get Dictionary Keys ${pel_records} 468 ${id}= Get From List ${ids} 0 469 Should Contain ${pel_records['${id}']['Sev']} Informational 470 471 472Verify Predictable Error Log 473 [Documentation] Create a predictive error and verify. 474 [Tags] Verify_Predictable_Error_Log 475 476 # Create a predictable error log. 477 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 478 ${pel_records}= Peltool -l 479 480 # An example of predictive error log data: 481 # { 482 # "0x5000069E": { 483 # "SRC": "BD8D1002", 484 # "Message": "An application had an internal failure", 485 # "PLID": "0x5000069E", 486 # "CreatorID": "BMC", 487 # "Subsystem": "BMC Firmware", 488 # "Commit Time": "10/14/2020 11:40:07", 489 # "Sev": "Predictive Error", 490 # "CompID": "0x1000" 491 # } 492 # } 493 494 ${pel_ids}= Get PEL Log Via BMC CLI 495 ${id}= Get From List ${pel_ids} -1 496 Should Contain ${pel_records['${id}']['Sev']} Predictive 497 498 499Verify Unrecoverable Error Log 500 [Documentation] Create an unrecoverable error and verify. 501 [Tags] Verify_Unrecoverable_Error_Log 502 503 # Create an internal failure error log. 504 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 505 ${pel_records}= Peltool -l 506 507 # An example of unrecoverable error log data: 508 # { 509 # "0x50000CC5": { 510 # "SRC": "BD8D1002", 511 # "Message": "An application had an internal failure", 512 # "PLID": "0x50000CC5", 513 # "CreatorID": "BMC", 514 # "Subsystem": "BMC Firmware", 515 # "Commit Time": "04/01/2020 16:44:55", 516 # "Sev": "Unrecoverable Error", 517 # "CompID": "0x1000" 518 # } 519 # } 520 521 ${pel_ids}= Get PEL Log Via BMC CLI 522 ${id}= Get From List ${pel_ids} -1 523 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 524 525 526Verify Error Logging Rotation Policy 527 [Documentation] Verify error logging rotation policy. 528 [Tags] Verify_Error_Logging_Rotation_Policy 529 [Template] Error Logging Rotation Policy 530 531 # Error logs to be created % of total logging space when error 532 # log exceeds max limit. 533 Informational BMC 3000 15 534 Predictive BMC 3000 30 535 Unrecoverable BMC 3000 30 536 Informational BMC 1500, Predictive BMC 1500 45 537 Informational BMC 1500, Unrecoverable BMC 1500 45 538 Unrecoverable BMC 1500, Predictive BMC 1500 30 539 540 541Verify Error Logging Rotation Policy With All Types Of Errors 542 [Documentation] Verify error logging rotation policy with all types of errors. 543 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Errors 544 [Template] Error Logging Rotation Policy 545 546 # Error logs to be created % of total logging space when error 547 # log exceeds max limit. 548 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 549 550 551Verify Error Logging Rotation Policy With HOST Error Logs 552 [Documentation] Verify error logging rotation policy for non bmc error logs. 553 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 554 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 555 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 556 [Template] Error Logging Rotation Policy 557 558 # Error logs to be created % of total logging space when error 559 # log exceeds max limit. 560 Unrecoverable HOST 3000 30 561 Unrecoverable HOST 1500, Informational BMC 1500 45 562 Unrecoverable HOST 1500, Predictive BMC 1500 60 563 564 565Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 566 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 567 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 568 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 569 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 570 [Template] Error Logging Rotation Policy 571 572 # Error logs to be created % of total logging space when error 573 # log exceeds max limit. 574 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 575 576 577Verify Old Logs Are Deleted When Count Crosses Max 578 [Documentation] Verify that when the count crosses max, older logs are deleted. 579 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 580 581 Redfish Purge Event Log 582 583 # Create 3000 error logs. 584 FOR ${count} IN RANGE ${3000} 585 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 586 END 587 588 # Retrieve the IDs of the logs. 589 ${pel_ids}= Get PEL Log Via BMC CLI 590 591 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 592 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 593 # Wait few seconds for error logs rotation to complete. 594 Sleep 10s 595 596 # Now verify that first log is no more available but the 3000th is available. 597 ${1st_id}= Get From List ${pel_ids} 0 598 ${3000th_id}= Get From List ${pel_ids} 3000 599 ${output}= peltool -i ${1st_id} 600 Should Contain ${output} PEL not found 601 ${output}= peltool -i ${3000th_id} 602 Should Not Contain ${output} PEL not found 603 604 605Verify Reverse Order Of PEL Logs 606 [Documentation] Verify PEL command to output PEL logs in reverse order. 607 [Tags] Verify_Reverse_PEL_Logs 608 609 Redfish Purge Event Log 610 611 # Below commands create unrecoverable error log at first and then the predictable error. 612 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 613 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 614 615 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 616 ${pel_records}= peltool -lr 617 618 # It is found that, variables like dictionary always keep items in sorted order that makes 619 # this verification not possible, hence json is used to keep the items original order. 620 ${pel_records}= Convert To String ${pel_records} 621 ${json_string}= Replace String ${pel_records} ' " 622 ${json_object}= Evaluate json.loads('''${json_string}''') json 623 624 ${list}= Convert To List ${json_object} 625 626 ${id1}= Get From List ${list} 0 627 ${id1}= Convert To Integer ${id1} 628 ${id2}= Get From List ${list} 1 629 ${id2}= Convert To Integer ${id2} 630 631 Should Be True ${id1} > ${id2} 632 633 634Verify Total PEL Count 635 [Documentation] Verify total PEL count returned by peltool command. 636 [Tags] Verify_Total_PEL_Count 637 638 # Initially remove all logs. 639 Redfish Purge Event Log 640 641 # Generate a random number between 1-20. 642 ${random}= Evaluate random.randint(1, 20) modules=random 643 644 # Generate predictive error log multiple times. 645 FOR ${count} IN RANGE 0 ${random} 646 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 647 END 648 649 # Check PEL log count via peltool command and compare it with actual generated log count. 650 ${pel_records}= peltool -n 651 652 Should Be Equal ${pel_records['Number of PELs found']} ${random} 653 654 655Verify Listing Information Error 656 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 657 [Tags] Verify_Listing_Information_Error 658 659 # Initially remove all logs. 660 Redfish Purge Event Log 661 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 662 663 # Generate informational logs and verify that it would not get listed by peltool's list command. 664 ${pel_records}= peltool -l 665 ${ids}= Get Dictionary Keys ${pel_records} 666 Should Be Empty ${ids} 667 668 # Verify that information logs get listed using peltool's list command with -lfh option. 669 ${pel_records}= peltool -lfh 670 ${ids}= Get Dictionary Keys ${pel_records} 671 Should Not Be Empty ${ids} 672 ${id}= Get From List ${ids} 0 673 Should Contain ${pel_records['${id}']['Sev']} Informational 674 675 676Verify PEL Delete 677 [Documentation] Verify that peltool command can delete PEL log based on id. 678 [Tags] Verify_PEL_Delete 679 680 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 681 ${pel_ids}= Get PEL Log Via BMC CLI 682 ${id}= Get From List ${pel_ids} -1 683 Peltool -d ${id} False 684 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 685 686 687*** Keywords *** 688 689Error Logging Rotation Policy 690 [Documentation] Verify that when maximum log limit is reached, given error logging type 691 ... are deleted when reached their max allocated space. 692 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 693 694 # Description of argument(s): 695 # error_log Error logs to be created (E.g. Informational BMC 3000 696 # stands for BMC created 3000 informational error logs). 697 # max_allocated_space_percentage The maximum percentage of disk usage for given error 698 # log type when maximum count/log size is reached. 699 # The maximum error log count is 3000. 700 701 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 702 # in BMC during the test. 703 Redfish Purge Event Log 704 705 @{lists}= Split String ${error_log_type} ,${SPACE} 706 707 ${length}= Get Length ${lists} 708 709 FOR ${list} IN RANGE ${length} 710 @{words}= Split String ${lists}[${list}] ${SPACE} 711 Create Error Log ${words}[0] ${words}[1] ${words}[2] 712 END 713 714 # Create an additional error log to exceed max error logs limit. 715 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 716 717 # Delay for BMC to perform delete older error logs when log limit exceeds. 718 Sleep 10s 719 720 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 721 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 722 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 723 724 ${disk_usage_percentage}= Get Disk Usage For Error Logs 725 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 726 ${percent_diff}= Evaluate abs(${percent_diff}) 727 Should Be True ${percent_diff} <= 0.5 728 729 730Create Error Log 731 [Arguments] ${error_severity} ${error_creator} ${count} 732 733 # Description of argument(s): 734 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 735 # error_creator Name of error log's creator(E.g BMC, Host Boot) 736 # count Number of error logs to be generated. 737 738 FOR ${i} IN RANGE 0 ${count} 739 ${cmd}= Set Variable If 740 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 741 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 742 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 743 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 744 BMC Execute Command ${cmd} 745 END 746 747 748Get Disk Usage For Error Logs 749 [Documentation] Get disk usage percentage for error logs. 750 751 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 752 753 ${usage_output}= Fetch From Left ${usage_output} \/ 754 755 # Covert disk usage unit from KB to MB. 756 ${usage_output}= Evaluate ${usage_output} / 1024 757 758 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 759 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 760 761 [return] ${usage_percent} 762 763 764Create Test PEL Log 765 [Documentation] Generate test PEL log. 766 [Arguments] ${pel_type}=Internal Failure 767 768 # Description of argument(s): 769 # pel_type The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout). 770 771 # Test PEL log entry example: 772 # { 773 # "0x5000002D": { 774 # "SRC": "BD8D1002", 775 # "Message": "An application had an internal failure", 776 # "PLID": "0x5000002D", 777 # "CreatorID": "BMC", 778 # "Subsystem": "BMC Firmware", 779 # "Commit Time": "02/25/2020 04:47:09", 780 # "Sev": "Unrecoverable Error", 781 # "CompID": "0x1000" 782 # } 783 # } 784 785 Run Keyword If '${pel_type}' == 'Internal Failure' 786 ... BMC Execute Command ${CMD_INTERNAL_FAILURE} 787 ... ELSE IF '${pel_type}' == 'FRU Callout' 788 ... BMC Execute Command ${CMD_FRU_CALLOUT} 789 ... ELSE IF '${pel_type}' == 'Procedure And Symbolic FRU Callout' 790 ... BMC Execute Command ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} 791 792 793Get PEL Log IDs 794 [Documentation] Returns the list of PEL log IDs which contains given field's value. 795 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 796 797 # Description of argument(s): 798 # pel_section The section of PEL (e.g. Private Header, User Header). 799 # pel_field The PEL field (e.g. Event Severity, Event Type). 800 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 801 802 ${pel_ids}= Get PEL Log Via BMC CLI 803 @{pel_id_list}= Create List 804 805 FOR ${id} IN @{pel_ids} 806 ${pel_output}= Peltool -i ${id} 807 # Example of PEL output from "peltool -i <id>" command. 808 # [Private Header]: 809 # [Created at]: 08/24/1928 12:04:06 810 # [Created by]: 0x584D 811 # [Sub-section type]: 0 812 # [Entry Id]: 0x50000BB7 813 # [Platform Log Id]: 0x8200061D 814 # [CSSVER]: 815 # [Section Version]: 1 816 # [Creator Subsystem]: PHYP 817 # [BMC Event Log Id]: 341 818 # [Committed at]: 03/25/1920 12:06:22 819 # [User Header]: 820 # [Log Committed by]: 0x4552 821 # [Action Flags]: 822 # [0]: Report Externally 823 # [Subsystem]: I/O Subsystem 824 # [Event Type]: Miscellaneous, Informational Only 825 # [Sub-section type]: 0 826 # [Event Scope]: Entire Platform 827 # [Event Severity]: Informational Event 828 # [Host Transmission]: Not Sent 829 # [Section Version]: 1 830 831 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 832 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 833 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 834 END 835 Sort List ${pel_id_list} 836 837 [Return] ${pel_id_list} 838 839 840Get PEL Log Via BMC CLI 841 [Documentation] Returns the list of PEL IDs using BMC CLI. 842 843 ${pel_records}= Peltool -l 844 ${ids}= Get Dictionary Keys ${pel_records} 845 Sort List ${ids} 846 847 [Return] ${ids} 848 849 850Get PEL Field Value 851 [Documentation] Returns the value of given PEL's field. 852 [Arguments] ${pel_id} ${pel_section} ${pel_field} 853 854 # Description of argument(s): 855 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 856 # pel_section The section of PEL (e.g. Private Header, User Header) 857 # pel_field The PEL field (e.g. Event Severity, Event Type). 858 859 ${pel_output}= Peltool -i ${pel_id} 860 861 # Example of PEL output from "peltool -i <id>" command. 862 # [Private Header]: 863 # [Created at]: 08/24/1928 12:04:06 864 # [Created by]: 0x584D 865 # [Sub-section type]: 0 866 # [Entry Id]: 0x50000BB7 867 # [Platform Log Id]: 0x8200061D 868 # [CSSVER]: 869 # [Section Version]: 1 870 # [Creator Subsystem]: PHYP 871 # [BMC Event Log Id]: 341 872 # [Committed at]: 03/25/1920 12:06:22 873 # [User Header]: 874 # [Log Committed by]: 0x4552 875 # [Action Flags]: 876 # [0]: Report Externally 877 # [Subsystem]: I/O Subsystem 878 # [Event Type]: Miscellaneous, Informational Only 879 # [Sub-section type]: 0 880 # [Event Scope]: Entire Platform 881 # [Event Severity]: Informational Event 882 # [Host Transmission]: Not Sent 883 # [Section Version]: 1 884 885 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 886 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 887 888 [Return] ${pel_field_output} 889