1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/openbmc_ffdc.robot 9 10Test Setup Redfish.Login 11Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 12 13 14*** Variables *** 15 16${CMD_INTERNAL_FAILURE} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 17... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 18... xyz.openbmc_project.Logging.Entry.Level.Error 0 19 20${CMD_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 21... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.Timeout 22... xyz.openbmc_project.Logging.Entry.Level.Error 2 "TIMEOUT_IN_MSEC" "5" 23... "CALLOUT_INVENTORY_PATH" "/xyz/openbmc_project/inventory/system/chassis/motherboard" 24 25${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 26... xyz.openbmc_project.Logging.Create Create ssa{ss} org.open_power.Logging.Error.TestError1 27... xyz.openbmc_project.Logging.Entry.Level.Error 0 28 29${CMD_INFORMATIONAL_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 30... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.TestError2 31... xyz.openbmc_project.Logging.Entry.Level.Informational 0 32 33${CMD_INVENTORY_PREFIX} busctl get-property xyz.openbmc_project.Inventory.Manager 34... /xyz/openbmc_project/inventory/system/chassis/motherboard 35 36${CMD_UNRECOVERABLE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 37... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 38... xyz.openbmc_project.Logging.Entry.Level.Error 0 39 40${CMD_PREDICTIVE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 41... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 42... xyz.openbmc_project.Logging.Entry.Level.Warning 0 43 44@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 45 46*** Test Cases *** 47 48Create Test PEL Log And Verify 49 [Documentation] Create PEL log using busctl command and verify via peltool. 50 [Tags] Create_Test_PEL_Log_And_Verify 51 52 Redfish Purge Event Log 53 Create Test PEL Log 54 ${pel_id}= Get PEL Log Via BMC CLI 55 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 56 57 58Verify PEL Log Details 59 [Documentation] Verify PEL log details via peltool. 60 [Tags] Verify_PEL_Log_Details 61 62 Redfish Purge Event Log 63 64 ${bmc_time1}= CLI Get BMC DateTime 65 Create Test PEL Log 66 ${bmc_time2}= CLI Get BMC DateTime 67 68 ${pel_records}= Peltool -l 69 70 # Example output from 'Peltool -l': 71 # pel_records: 72 # [0x50000012]: 73 # [CreatorID]: BMC 74 # [CompID]: 0x1000 75 # [PLID]: 0x50000012 76 # [Subsystem]: BMC Firmware 77 # [Message]: An application had an internal failure 78 # [SRC]: BD8D1002 79 # [Commit Time]: 03/02/2020 09:35:15 80 # [Sev]: Unrecoverable Error 81 82 ${ids}= Get Dictionary Keys ${pel_records} 83 ${id}= Get From List ${ids} 0 84 85 @{pel_fields}= Create List CreatorID Subsystem Message Sev 86 FOR ${field} IN @{pel_fields} 87 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 88 END 89 90 Valid Value pel_records['${id}']['PLID'] ['${id}'] 91 92 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 93 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 94 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 95 96 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 97 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 98 99 # Convert BMC and PEL time to epoch time before comparing. 100 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 101 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 102 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 103 104 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 105 106 107Verify Mandatory Sections Of Error Log PEL 108 [Documentation] Verify mandatory sections of error log PEL. 109 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 110 111 Create Test PEL Log 112 113 ${pel_ids}= Get PEL Log Via BMC CLI 114 ${pel_id}= Get From List ${pel_ids} -1 115 ${pel_output}= Peltool -i ${pel_id} 116 ${pel_sections}= Get Dictionary Keys ${pel_output} 117 118 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 119 120 121Verify PEL Log Persistence After BMC Reboot 122 [Documentation] Verify PEL log persistence after BMC reboot. 123 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 124 125 Create Test PEL Log 126 ${pel_before_reboot}= Get PEL Log Via BMC CLI 127 128 Redfish OBMC Reboot (off) 129 ${pel_after_reboot}= Get PEL Log Via BMC CLI 130 131 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 132 133 134Verify PEL ID Numbering 135 [Documentation] Verify PEL ID numbering. 136 [Tags] Verify_PEL_ID_Numbering 137 138 Redfish Purge Event Log 139 Create Test PEL Log 140 Create Test PEL Log 141 142 ${pel_ids}= Get PEL Log Via BMC CLI 143 144 # Example of PEL IDs from PEL logs. 145 # [0x50000012]: <--- First PEL ID 146 # [CreatorID]: BMC 147 # [CompID]: 0x1000 148 # [PLID]: 0x50000012 149 # [Subsystem]: BMC Firmware 150 # [Message]: An application had an internal failure 151 # [SRC]: BD8D1002 152 # [Commit Time]: 03/02/2020 09:35:15 153 # [Sev]: Unrecoverable Error 154 # 155 # [0x50000013]: <--- Second PEL ID 156 # [CreatorID]: BMC 157 # [CompID]: 0x1000 158 # [PLID]: 0x50000013 159 # [Subsystem]: BMC Firmware 160 # [Message]: An application had an internal failure 161 # [SRC]: BD8D1002 162 # [Commit Time]: 03/02/2020 09:35:15 163 # [Sev]: Unrecoverable Error 164 165 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 166 167Verify Machine Type Model And Serial Number 168 [Documentation] Verify machine type model and serial number from PEL. 169 [Tags] Verify_Machine_Type_Model_And_Serial_Number 170 171 Create Test PEL Log 172 173 ${pel_ids}= Get PEL Log Via BMC CLI 174 ${id}= Get From List ${pel_ids} -1 175 176 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 177 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 178 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 179 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 180 181 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 182 # [Failing MTMS]: 183 # [Created by]: 0x2000 184 # [Machine Type Model]: 1234-ABC <---- Machine type 185 # [Section Version]: 1 186 # [Serial Number]: ABCDEFG <---- Serial number 187 # [Sub-section type]: 0 188 189 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 190 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 191 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 192 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 193 194 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 195 Valid Value pel_serial_number ['${redfish_serial_number}'] 196 197 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 198 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 199 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 200 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 201 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 202 203 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 204 Valid Value pel_serial_number ['${redfish_serial_number}'] 205 206 207Verify Host Off State From PEL 208 [Documentation] Verify Host off state from PEL. 209 [Tags] Verify_Host_Off_State_From_PEL 210 211 Redfish Power Off stack_mode=skip 212 Create Test PEL Log 213 214 ${pel_ids}= Get PEL Log Via BMC CLI 215 ${id}= Get From List ${pel_ids} -1 216 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 217 218 Valid Value pel_host_state ['Off'] 219 220 221Verify BMC Version From PEL 222 [Documentation] Verify BMC Version from PEL. 223 [Tags] Verify_BMC_Version_From_PEL 224 225 Create Test PEL Log 226 227 ${pel_ids}= Get PEL Log Via BMC CLI 228 ${id}= Get From List ${pel_ids} -1 229 ${pel_bmc_version}= Get PEL Field Value ${id} User Data BMC Version ID 230 231 ${bmc_version}= Get BMC Version 232 Valid Value bmc_version ['${bmc_version}'] 233 234 235Verify PEL Log After Host Poweron 236 [Documentation] Verify PEL log generation while booting host. 237 [Tags] Verify_PEL_Log_After_Host_Poweron 238 239 Redfish Power Off stack_mode=skip 240 Redfish Purge Event Log 241 Redfish Power On stack_mode=skip 242 243 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 244 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 245 246 # Get BMC created non-infomational error. 247 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 248 249 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 250 251 252Verify BMC Event Log ID 253 [Documentation] Verify BMC Event Log ID from PEL. 254 [Tags] Verify_BMC_Event_Log_ID 255 256 Redfish Purge Event Log 257 Create Test PEL Log 258 259 ${pel_ids}= Get PEL Log Via BMC CLI 260 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 261 262 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 263 # [Private Header]: 264 # [Created at]: 08/24/1928 12:04:06 265 # [Created by]: 0x584D 266 # [Sub-section type]: 0 267 # [Entry Id]: 0x50000BB7 268 # [Platform Log Id]: 0x8200061D 269 # [CSSVER]: 270 # [Section Version]: 1 271 # [Creator Subsystem]: PHYP 272 # [BMC Event Log Id]: 341 <---- BMC event log id value 273 # [Committed at]: 03/25/1920 12:06:22 274 275 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 276 277 # Example of redfish_event_logs output: 278 # redfish_event_logs: 279 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 280 # [Name]: System Event Log Entries 281 # [Members@odata.count]: 1 282 # [@odata.type]: #LogEntryCollection.LogEntryCollection 283 # [Description]: Collection of System Event Log Entries 284 # [Members]: 285 # [0]: 286 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 287 # [Name]: System Event Log Entry 288 # [Severity]: Critical 289 # [EntryType]: Event 290 # [Created]: 2020-04-02T07:25:13+00:00 291 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 292 # [Id]: 235 <----- Event log ID 293 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 294 295 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 296 297 298Verify FRU Callout 299 [Documentation] Verify FRU callout entries from PEL log. 300 [Tags] Verify_FRU_Callout 301 302 Create Test PEL Log FRU Callout 303 304 ${pel_ids}= Get PEL Log Via BMC CLI 305 ${id}= Get From List ${pel_ids} -1 306 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 307 308 # Example of PEL Callout Section from "peltool -i <id>" command. 309 # [Callouts]: 310 # [0]: 311 # [FRU Type]: Normal Hardware FRU 312 # [Priority]: Mandatory, replace all with this type as a unit 313 # [Location Code]: U78DA.ND1.1234567-P0 314 # [Part Number]: F191014 315 # [CCIN]: 2E2D 316 # [Serial Number]: YL2E2D010000 317 # [Callout Count]: 1 318 319 Valid Value pel_callout_section['Callout Count'] ['1'] 320 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 321 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 322 323 # Verify Location Code field of PEL callout with motherboard's Location Code. 324 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 325 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 326 ... ${busctl_output[0].split('"')[1].strip('"')} 327 328 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 329 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 330 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 331 332 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 333 ${busctl_output}= BMC Execute Command 334 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 335 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 336 ... ${busctl_output[0].split('"')[1].strip('"')} 337 338 339Verify Procedure And Symbolic FRU Callout 340 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 341 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 342 343 Create Test PEL Log Procedure And Symbolic FRU Callout 344 345 ${pel_ids}= Get PEL Log Via BMC CLI 346 ${id}= Get From List ${pel_ids} -1 347 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 348 349 # Example of PEL Callout Section from "peltool -i <id>" command. 350 # [Callouts]: 351 # [0]: 352 # [Priority]: Mandatory, replace all with this type as a unit 353 # [Procedure Number]: BMCSP02 354 # [FRU Type]: Maintenance Procedure Required 355 # [1]: 356 # [Priority]: Medium Priority 357 # [Part Number]: SVCDOCS 358 # [FRU Type]: Symbolic FRU 359 # [Callout Count]: 2 360 361 Valid Value pel_callout_section['Callout Count'] ['2'] 362 363 # Verify procedural callout info. 364 365 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 366 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 367 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 368 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 369 370 # Verify procedural callout info. 371 372 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 373 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 374 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 375 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 376 377 378Verify PEL Log Entry For Event Log 379 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 380 [Tags] Verify_PEL_Log_Entry_For_Event_Log 381 382 Redfish Purge Event Log 383 # Create an internal failure error log. 384 BMC Execute Command ${CMD_INTERNAL_FAILURE} 385 386 ${elog_entry}= Get Event Logs 387 # Example of Redfish event logs: 388 # elog_entry: 389 # [0]: 390 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 391 # [Created]: 2020-04-20T01:55:22+00:00 392 # [Id]: 1 393 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 394 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 395 # [EntryType]: Event 396 # [Severity]: Critical 397 # [Name]: System Event Log Entry 398 399 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 400 401 ${pel_records}= Peltool -l 402 # Example output from 'Peltool -l': 403 # pel_records: 404 # [0x50000023]: 405 # [SRC]: BD8D1002 406 # [CreatorID]: BMC 407 # [Message]: An application had an internal failure 408 # [CompID]: 0x1000 409 # [PLID]: 0x50000023 410 # [Commit Time]: 04/20/2020 01:55:22 411 # [Subsystem]: BMC Firmware 412 # [Sev]: Unrecoverable Error 413 414 ${ids}= Get Dictionary Keys ${pel_records} 415 ${id}= Get From List ${ids} 0 416 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 417 ... date_format=%m/%d/%Y %H:%M:%S 418 419 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 420 Should Contain Any ${pel_records['${id}']['Message']} internal failure ignore_case=True 421 Should Contain Any ${elog_entry[0]['Message']} InternalFailure ignore_case=True 422 423 Should Be Equal ${redfish_log_time} ${pel_log_time} 424 425 426Verify Delete All PEL 427 [Documentation] Verify deleting all PEL logs. 428 [Tags] Verify_Delete_All_PEL 429 430 Create Test PEL Log 431 Create Test PEL Log 432 Peltool --delete-all False 433 434 ${pel_ids}= Get PEL Log Via BMC CLI 435 Should Be Empty ${pel_ids} 436 437 438Verify Informational Error Log 439 [Documentation] Create an informational error log and verify. 440 [Tags] Verify_Informational_Error_Log 441 442 Redfish Purge Event Log 443 # Create an informational error log. 444 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 445 ${pel_records}= Peltool -lfh 446 447 # An example of information error log data: 448 # { 449 # "0x500006A0": { 450 # "SRC": "BD8D1002", 451 # "Message": "An application had an internal failure", 452 # "PLID": "0x500006A0", 453 # "CreatorID": "BMC", 454 # "Subsystem": "BMC Firmware", 455 # "Commit Time": "10/14/2020 11:41:38", 456 # "Sev": "Informational Event", 457 # "CompID": "0x1000" 458 # } 459 # } 460 461 ${ids}= Get Dictionary Keys ${pel_records} 462 ${id}= Get From List ${ids} 0 463 Should Contain ${pel_records['${id}']['Sev']} Informational 464 465 466Verify Predictable Error Log 467 [Documentation] Create a predictive error and verify. 468 [Tags] Verify_Predictable_Error_Log 469 470 # Create a predictable error log. 471 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 472 ${pel_records}= Peltool -l 473 474 # An example of predictive error log data: 475 # { 476 # "0x5000069E": { 477 # "SRC": "BD8D1002", 478 # "Message": "An application had an internal failure", 479 # "PLID": "0x5000069E", 480 # "CreatorID": "BMC", 481 # "Subsystem": "BMC Firmware", 482 # "Commit Time": "10/14/2020 11:40:07", 483 # "Sev": "Predictive Error", 484 # "CompID": "0x1000" 485 # } 486 # } 487 488 ${pel_ids}= Get PEL Log Via BMC CLI 489 ${id}= Get From List ${pel_ids} -1 490 Should Contain ${pel_records['${id}']['Sev']} Predictive 491 492 493Verify Unrecoverable Error Log 494 [Documentation] Create an unrecoverable error and verify. 495 [Tags] Verify_Unrecoverable_Error_Log 496 497 # Create an internal failure error log. 498 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 499 ${pel_records}= Peltool -l 500 501 # An example of unrecoverable error log data: 502 # { 503 # "0x50000CC5": { 504 # "SRC": "BD8D1002", 505 # "Message": "An application had an internal failure", 506 # "PLID": "0x50000CC5", 507 # "CreatorID": "BMC", 508 # "Subsystem": "BMC Firmware", 509 # "Commit Time": "04/01/2020 16:44:55", 510 # "Sev": "Unrecoverable Error", 511 # "CompID": "0x1000" 512 # } 513 # } 514 515 ${pel_ids}= Get PEL Log Via BMC CLI 516 ${id}= Get From List ${pel_ids} -1 517 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 518 519 520Verify Error Logging Rotation Policy 521 [Documentation] Verify error logging rotation policy. 522 [Tags] Verify_Error_Logging_Rotation_Policy 523 [Template] Error Logging Rotation Policy 524 525 # Error logs to be created % of total logging space when error 526 # log exceeds max limit. 527 Informational BMC 3000 15 528 Predictive BMC 3000 30 529 Unrecoverable BMC 3000 30 530 Informational BMC 1500, Predictive BMC 1500 45 531 Informational BMC 1500, Unrecoverable BMC 1500 45 532 Unrecoverable BMC 1500, Predictive BMC 1500 30 533 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 534 535Verify Reverse Order Of PEL Logs 536 [Documentation] Verify PEL command to output PEL logs in reverse order. 537 [Tags] Verify_Reverse_PEL_Logs 538 539 Redfish Purge Event Log 540 541 # Below commands create unrecoverable error log at first and then the predictable error. 542 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 543 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 544 545 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 546 ${pel_records}= peltool -lr 547 548 # It is found that, variables like dictionary always keep items in sorted order that makes 549 # this verification not possible, hence json is used to keep the items original order. 550 ${pel_records}= Convert To String ${pel_records} 551 ${json_string}= Replace String ${pel_records} ' " 552 ${json_object}= Evaluate json.loads('''${json_string}''') json 553 554 ${list}= Convert To List ${json_object} 555 556 ${id1}= Get From List ${list} 0 557 ${id1}= Convert To Integer ${id1} 558 ${id2}= Get From List ${list} 1 559 ${id2}= Convert To Integer ${id2} 560 561 Should Be True ${id1} > ${id2} 562 563 564Verify Total PEL Count 565 [Documentation] Verify total PEL count returned by peltool command. 566 [Tags] Verify_Total_PEL_Count 567 568 # Initially remove all logs. 569 Redfish Purge Event Log 570 571 # Generate a random number between 1-20. 572 ${random}= Evaluate random.randint(1, 20) modules=random 573 574 # Generate predictive error log multiple times. 575 FOR ${count} IN RANGE 0 ${random} 576 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 577 END 578 579 # Check PEL log count via peltool command and compare it with actual generated log count. 580 ${pel_records}= peltool -n 581 582 Should Be Equal ${pel_records['Number of PELs found']} ${random} 583 584 585Verify Listing Information Error 586 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 587 [Tags] Verify_Listing_Information_Error 588 589 # Initially remove all logs. 590 Redfish Purge Event Log 591 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 592 593 # Generate informational logs and verify that it would not get listed by peltool's list command. 594 ${pel_records}= peltool -l 595 ${ids}= Get Dictionary Keys ${pel_records} 596 Should Be Empty ${ids} 597 598 # Verify that information logs get listed using peltool's list command with -lfh option. 599 ${pel_records}= peltool -lfh 600 ${ids}= Get Dictionary Keys ${pel_records} 601 Should Not Be Empty ${ids} 602 ${id}= Get From List ${ids} 0 603 Should Contain ${pel_records['${id}']['Sev']} Informational 604 605 606Verify PEL Delete 607 [Documentation] Verify that peltool command can delete PEL log based on id. 608 [Tags] Verify_PEL_Delete 609 610 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 611 ${pel_ids}= Get PEL Log Via BMC CLI 612 ${id}= Get From List ${pel_ids} -1 613 Peltool -d ${id} 614 ${output}= peltool -i ${id} 615 Should Contain ${output} PEL not found 616 617 618*** Keywords *** 619 620Error Logging Rotation Policy 621 [Documentation] Verify that when maximum log limit is reached, given error logging type 622 ... are deleted when reached their max allocated space. 623 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 624 625 # Description of argument(s): 626 # error_log Error logs to be created (E.g. Informational BMC 3000 627 # stands for BMC created 3000 informational error logs). 628 # max_allocated_space_percentage The maximum percentage of disk usage for given error 629 # log type when maximum count/log size is reached. 630 # The maximum error log count is 3000. 631 632 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 633 # in BMC during the test. 634 Redfish Purge Event Log 635 636 @{lists}= Split String ${error_log_type} ,${SPACE} 637 638 ${length}= Get Length ${lists} 639 640 FOR ${list} IN RANGE ${length} 641 @{words}= Split String ${lists}[${list}] ${SPACE} 642 Create Error Log ${words}[0] ${words}[1] ${words}[2] 643 END 644 645 # Create an additional error log to exceed max error logs limit. 646 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 647 648 # Delay for BMC to perform delete older error logs when log limit exceeds. 649 Sleep 10s 650 651 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 652 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 653 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 654 655 ${disk_usage_percentage}= Get Disk Usage For Error Logs 656 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 657 ${percent_diff}= Evaluate abs(${percent_diff}) 658 Should Be True ${percent_diff} <= 0.5 659 660 661Create Error Log 662 [Arguments] ${error_severity} ${error_creator} ${count} 663 664 # Description of argument(s): 665 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 666 # error_creator Name of error log's creator(E.g BMC, Host Boot) 667 # count Number of error logs to be generated. 668 669 FOR ${i} IN RANGE 0 ${count} 670 ${cmd}= Set Variable If 671 ... '${error_severity}' == 'Informational' ${CMD_INFORMATIONAL_ERROR} 672 ... '${error_severity}' == 'Predictive' ${CMD_PREDICTIVE_ERROR} 673 ... '${error_severity}' == 'Unrecoverable' ${CMD_UNRECOVERABLE_ERROR} 674 BMC Execute Command ${cmd} 675 END 676 677 678Get Disk Usage For Error Logs 679 [Documentation] Get disk usage percentage for error logs. 680 681 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 682 683 ${usage_output}= Fetch From Left ${usage_output} \/ 684 685 # Covert disk usage unit from KB to MB. 686 ${usage_output}= Evaluate ${usage_output} / 1024 687 688 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 689 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 690 691 [return] ${usage_percent} 692 693 694Create Test PEL Log 695 [Documentation] Generate test PEL log. 696 [Arguments] ${pel_type}=Internal Failure 697 698 # Description of argument(s): 699 # pel_type The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout). 700 701 # Test PEL log entry example: 702 # { 703 # "0x5000002D": { 704 # "SRC": "BD8D1002", 705 # "Message": "An application had an internal failure", 706 # "PLID": "0x5000002D", 707 # "CreatorID": "BMC", 708 # "Subsystem": "BMC Firmware", 709 # "Commit Time": "02/25/2020 04:47:09", 710 # "Sev": "Unrecoverable Error", 711 # "CompID": "0x1000" 712 # } 713 # } 714 715 Run Keyword If '${pel_type}' == 'Internal Failure' 716 ... BMC Execute Command ${CMD_INTERNAL_FAILURE} 717 ... ELSE IF '${pel_type}' == 'FRU Callout' 718 ... BMC Execute Command ${CMD_FRU_CALLOUT} 719 ... ELSE IF '${pel_type}' == 'Procedure And Symbolic FRU Callout' 720 ... BMC Execute Command ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} 721 722 723Get PEL Log IDs 724 [Documentation] Returns the list of PEL log IDs which contains given field's value. 725 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 726 727 # Description of argument(s): 728 # pel_section The section of PEL (e.g. Private Header, User Header). 729 # pel_field The PEL field (e.g. Event Severity, Event Type). 730 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 731 732 ${pel_ids}= Get PEL Log Via BMC CLI 733 @{pel_id_list}= Create List 734 735 FOR ${id} IN @{pel_ids} 736 ${pel_output}= Peltool -i ${id} 737 # Example of PEL output from "peltool -i <id>" command. 738 # [Private Header]: 739 # [Created at]: 08/24/1928 12:04:06 740 # [Created by]: 0x584D 741 # [Sub-section type]: 0 742 # [Entry Id]: 0x50000BB7 743 # [Platform Log Id]: 0x8200061D 744 # [CSSVER]: 745 # [Section Version]: 1 746 # [Creator Subsystem]: PHYP 747 # [BMC Event Log Id]: 341 748 # [Committed at]: 03/25/1920 12:06:22 749 # [User Header]: 750 # [Log Committed by]: 0x4552 751 # [Action Flags]: 752 # [0]: Report Externally 753 # [Subsystem]: I/O Subsystem 754 # [Event Type]: Miscellaneous, Informational Only 755 # [Sub-section type]: 0 756 # [Event Scope]: Entire Platform 757 # [Event Severity]: Informational Event 758 # [Host Transmission]: Not Sent 759 # [Section Version]: 1 760 761 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 762 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 763 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 764 END 765 Sort List ${pel_id_list} 766 767 [Return] ${pel_id_list} 768 769 770Get PEL Log Via BMC CLI 771 [Documentation] Returns the list of PEL IDs using BMC CLI. 772 773 ${pel_records}= Peltool -l 774 ${ids}= Get Dictionary Keys ${pel_records} 775 Sort List ${ids} 776 777 [Return] ${ids} 778 779 780Get PEL Field Value 781 [Documentation] Returns the value of given PEL's field. 782 [Arguments] ${pel_id} ${pel_section} ${pel_field} 783 784 # Description of argument(s): 785 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 786 # pel_section The section of PEL (e.g. Private Header, User Header) 787 # pel_field The PEL field (e.g. Event Severity, Event Type). 788 789 ${pel_output}= Peltool -i ${pel_id} 790 791 # Example of PEL output from "peltool -i <id>" command. 792 # [Private Header]: 793 # [Created at]: 08/24/1928 12:04:06 794 # [Created by]: 0x584D 795 # [Sub-section type]: 0 796 # [Entry Id]: 0x50000BB7 797 # [Platform Log Id]: 0x8200061D 798 # [CSSVER]: 799 # [Section Version]: 1 800 # [Creator Subsystem]: PHYP 801 # [BMC Event Log Id]: 341 802 # [Committed at]: 03/25/1920 12:06:22 803 # [User Header]: 804 # [Log Committed by]: 0x4552 805 # [Action Flags]: 806 # [0]: Report Externally 807 # [Subsystem]: I/O Subsystem 808 # [Event Type]: Miscellaneous, Informational Only 809 # [Sub-section type]: 0 810 # [Event Scope]: Entire Platform 811 # [Event Severity]: Informational Event 812 # [Host Transmission]: Not Sent 813 # [Section Version]: 1 814 815 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 816 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 817 818 [Return] ${pel_field_output} 819