1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/openbmc_ffdc.robot 9 10Test Setup Redfish.Login 11Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 12 13 14*** Variables *** 15 16${CMD_INTERNAL_FAILURE} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 17... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 18... xyz.openbmc_project.Logging.Entry.Level.Error 0 19 20${CMD_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 21... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.Timeout 22... xyz.openbmc_project.Logging.Entry.Level.Error 2 "TIMEOUT_IN_MSEC" "5" 23... "CALLOUT_INVENTORY_PATH" "/xyz/openbmc_project/inventory/system/chassis/motherboard" 24 25${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 26... xyz.openbmc_project.Logging.Create Create ssa{ss} org.open_power.Logging.Error.TestError1 27... xyz.openbmc_project.Logging.Entry.Level.Error 0 28 29${CMD_INFORMATIONAL_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 30... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.TestError2 31... xyz.openbmc_project.Logging.Entry.Level.Informational 0 32 33${CMD_INVENTORY_PREFIX} busctl get-property xyz.openbmc_project.Inventory.Manager 34... /xyz/openbmc_project/inventory/system/chassis/motherboard 35 36${CMD_UNRECOVERABLE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 37... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 38... xyz.openbmc_project.Logging.Entry.Level.Error 0 39 40${CMD_PREDICTIVE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 41... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 42... xyz.openbmc_project.Logging.Entry.Level.Warning 0 43 44@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 45 46*** Test Cases *** 47 48Create Test PEL Log And Verify 49 [Documentation] Create PEL log using busctl command and verify via peltool. 50 [Tags] Create_Test_PEL_Log_And_Verify 51 52 Redfish Purge Event Log 53 Create Test PEL Log 54 ${pel_id}= Get PEL Log Via BMC CLI 55 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 56 57 58Verify PEL Log Details 59 [Documentation] Verify PEL log details via peltool. 60 [Tags] Verify_PEL_Log_Details 61 62 Redfish Purge Event Log 63 64 ${bmc_time1}= CLI Get BMC DateTime 65 Create Test PEL Log 66 ${bmc_time2}= CLI Get BMC DateTime 67 68 ${pel_records}= Peltool -l 69 70 # Example output from 'Peltool -l': 71 # pel_records: 72 # [0x50000012]: 73 # [CreatorID]: BMC 74 # [CompID]: 0x1000 75 # [PLID]: 0x50000012 76 # [Subsystem]: BMC Firmware 77 # [Message]: An application had an internal failure 78 # [SRC]: BD8D1002 79 # [Commit Time]: 03/02/2020 09:35:15 80 # [Sev]: Unrecoverable Error 81 82 ${ids}= Get Dictionary Keys ${pel_records} 83 ${id}= Get From List ${ids} 0 84 85 @{pel_fields}= Create List CreatorID Subsystem Message Sev 86 FOR ${field} IN @{pel_fields} 87 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 88 END 89 90 Valid Value pel_records['${id}']['PLID'] ['${id}'] 91 92 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 93 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 94 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 95 96 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 97 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 98 99 # Convert BMC and PEL time to epoch time before comparing. 100 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 101 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 102 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 103 104 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 105 106 107Verify Mandatory Sections Of Error Log PEL 108 [Documentation] Verify mandatory sections of error log PEL. 109 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 110 111 Create Test PEL Log 112 113 ${pel_ids}= Get PEL Log Via BMC CLI 114 ${pel_id}= Get From List ${pel_ids} -1 115 ${pel_output}= Peltool -i ${pel_id} 116 ${pel_sections}= Get Dictionary Keys ${pel_output} 117 118 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 119 120 121Verify PEL Log Persistence After BMC Reboot 122 [Documentation] Verify PEL log persistence after BMC reboot. 123 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 124 125 Create Test PEL Log 126 ${pel_before_reboot}= Get PEL Log Via BMC CLI 127 128 Redfish OBMC Reboot (off) 129 ${pel_after_reboot}= Get PEL Log Via BMC CLI 130 131 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 132 133 134Verify PEL ID Numbering 135 [Documentation] Verify PEL ID numbering. 136 [Tags] Verify_PEL_ID_Numbering 137 138 Redfish Purge Event Log 139 Create Test PEL Log 140 Create Test PEL Log 141 142 ${pel_ids}= Get PEL Log Via BMC CLI 143 144 # Example of PEL IDs from PEL logs. 145 # [0x50000012]: <--- First PEL ID 146 # [CreatorID]: BMC 147 # [CompID]: 0x1000 148 # [PLID]: 0x50000012 149 # [Subsystem]: BMC Firmware 150 # [Message]: An application had an internal failure 151 # [SRC]: BD8D1002 152 # [Commit Time]: 03/02/2020 09:35:15 153 # [Sev]: Unrecoverable Error 154 # 155 # [0x50000013]: <--- Second PEL ID 156 # [CreatorID]: BMC 157 # [CompID]: 0x1000 158 # [PLID]: 0x50000013 159 # [Subsystem]: BMC Firmware 160 # [Message]: An application had an internal failure 161 # [SRC]: BD8D1002 162 # [Commit Time]: 03/02/2020 09:35:15 163 # [Sev]: Unrecoverable Error 164 165 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 166 167Verify Machine Type Model And Serial Number 168 [Documentation] Verify machine type model and serial number from PEL. 169 [Tags] Verify_Machine_Type_Model_And_Serial_Number 170 171 Create Test PEL Log 172 173 ${pel_ids}= Get PEL Log Via BMC CLI 174 ${id}= Get From List ${pel_ids} -1 175 176 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 177 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 178 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 179 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 180 181 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 182 # [Failing MTMS]: 183 # [Created by]: 0x2000 184 # [Machine Type Model]: 1234-ABC <---- Machine type 185 # [Section Version]: 1 186 # [Serial Number]: ABCDEFG <---- Serial number 187 # [Sub-section type]: 0 188 189 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 190 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 191 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 192 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 193 194 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 195 Valid Value pel_serial_number ['${redfish_serial_number}'] 196 197 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 198 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 199 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 200 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 201 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 202 203 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 204 Valid Value pel_serial_number ['${redfish_serial_number}'] 205 206 207Verify Host Off State From PEL 208 [Documentation] Verify Host off state from PEL. 209 [Tags] Verify_Host_Off_State_From_PEL 210 211 Redfish Power Off stack_mode=skip 212 Create Test PEL Log 213 214 ${pel_ids}= Get PEL Log Via BMC CLI 215 ${id}= Get From List ${pel_ids} -1 216 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 217 218 Valid Value pel_host_state ['Off'] 219 220 221Verify BMC Version From PEL 222 [Documentation] Verify BMC Version from PEL. 223 [Tags] Verify_BMC_Version_From_PEL 224 225 Create Test PEL Log 226 227 ${pel_ids}= Get PEL Log Via BMC CLI 228 ${id}= Get From List ${pel_ids} -1 229 ${pel_bmc_version}= Get PEL Field Value ${id} User Data BMC Version ID 230 231 ${bmc_version}= Get BMC Version 232 Valid Value bmc_version ['${bmc_version}'] 233 234 235Verify PEL Log After Host Poweron 236 [Documentation] Verify PEL log generation while booting host. 237 [Tags] Verify_PEL_Log_After_Host_Poweron 238 239 Redfish Power Off stack_mode=skip 240 Redfish Purge Event Log 241 Redfish Power On stack_mode=skip 242 243 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 244 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 245 246 # Get BMC created non-infomational error. 247 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 248 249 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 250 251 252Verify BMC Event Log ID 253 [Documentation] Verify BMC Event Log ID from PEL. 254 [Tags] Verify_BMC_Event_Log_ID 255 256 Redfish Purge Event Log 257 Create Test PEL Log 258 259 ${pel_ids}= Get PEL Log Via BMC CLI 260 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 261 262 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 263 # [Private Header]: 264 # [Created at]: 08/24/1928 12:04:06 265 # [Created by]: 0x584D 266 # [Sub-section type]: 0 267 # [Entry Id]: 0x50000BB7 268 # [Platform Log Id]: 0x8200061D 269 # [CSSVER]: 270 # [Section Version]: 1 271 # [Creator Subsystem]: PHYP 272 # [BMC Event Log Id]: 341 <---- BMC event log id value 273 # [Committed at]: 03/25/1920 12:06:22 274 275 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 276 277 # Example of redfish_event_logs output: 278 # redfish_event_logs: 279 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 280 # [Name]: System Event Log Entries 281 # [Members@odata.count]: 1 282 # [@odata.type]: #LogEntryCollection.LogEntryCollection 283 # [Description]: Collection of System Event Log Entries 284 # [Members]: 285 # [0]: 286 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 287 # [Name]: System Event Log Entry 288 # [Severity]: Critical 289 # [EntryType]: Event 290 # [Created]: 2020-04-02T07:25:13+00:00 291 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 292 # [Id]: 235 <----- Event log ID 293 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 294 295 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 296 297 298Verify FRU Callout 299 [Documentation] Verify FRU callout entries from PEL log. 300 [Tags] Verify_FRU_Callout 301 302 Create Test PEL Log FRU Callout 303 304 ${pel_ids}= Get PEL Log Via BMC CLI 305 ${id}= Get From List ${pel_ids} -1 306 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 307 308 # Example of PEL Callout Section from "peltool -i <id>" command. 309 # [Callouts]: 310 # [0]: 311 # [FRU Type]: Normal Hardware FRU 312 # [Priority]: Mandatory, replace all with this type as a unit 313 # [Location Code]: U78DA.ND1.1234567-P0 314 # [Part Number]: F191014 315 # [CCIN]: 2E2D 316 # [Serial Number]: YL2E2D010000 317 # [Callout Count]: 1 318 319 Valid Value pel_callout_section['Callout Count'] ['1'] 320 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 321 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 322 323 # Verify Location Code field of PEL callout with motherboard's Location Code. 324 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 325 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 326 ... ${busctl_output[0].split('"')[1].strip('"')} 327 328 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 329 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 330 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 331 332 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 333 ${busctl_output}= BMC Execute Command 334 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 335 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 336 ... ${busctl_output[0].split('"')[1].strip('"')} 337 338 339Verify Procedure And Symbolic FRU Callout 340 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 341 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 342 343 Create Test PEL Log Procedure And Symbolic FRU Callout 344 345 ${pel_ids}= Get PEL Log Via BMC CLI 346 ${id}= Get From List ${pel_ids} -1 347 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 348 349 # Example of PEL Callout Section from "peltool -i <id>" command. 350 # [Callouts]: 351 # [0]: 352 # [Priority]: Mandatory, replace all with this type as a unit 353 # [Procedure Number]: BMCSP02 354 # [FRU Type]: Maintenance Procedure Required 355 # [1]: 356 # [Priority]: Medium Priority 357 # [Part Number]: SVCDOCS 358 # [FRU Type]: Symbolic FRU 359 # [Callout Count]: 2 360 361 Valid Value pel_callout_section['Callout Count'] ['2'] 362 363 # Verify procedural callout info. 364 365 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 366 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 367 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 368 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 369 370 # Verify procedural callout info. 371 372 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 373 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 374 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 375 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 376 377 378Verify PEL Log Entry For Event Log 379 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 380 [Tags] Verify_PEL_Log_Entry_For_Event_Log 381 382 Redfish Purge Event Log 383 # Create an internal failure error log. 384 BMC Execute Command ${CMD_INTERNAL_FAILURE} 385 386 ${elog_entry}= Get Event Logs 387 # Example of Redfish event logs: 388 # elog_entry: 389 # [0]: 390 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 391 # [Created]: 2020-04-20T01:55:22+00:00 392 # [Id]: 1 393 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 394 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 395 # [EntryType]: Event 396 # [Severity]: Critical 397 # [Name]: System Event Log Entry 398 399 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 400 401 ${pel_records}= Peltool -l 402 # Example output from 'Peltool -l': 403 # pel_records: 404 # [0x50000023]: 405 # [SRC]: BD8D1002 406 # [CreatorID]: BMC 407 # [Message]: An application had an internal failure 408 # [CompID]: 0x1000 409 # [PLID]: 0x50000023 410 # [Commit Time]: 04/20/2020 01:55:22 411 # [Subsystem]: BMC Firmware 412 # [Sev]: Unrecoverable Error 413 414 ${ids}= Get Dictionary Keys ${pel_records} 415 ${id}= Get From List ${ids} 0 416 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 417 ... date_format=%m/%d/%Y %H:%M:%S 418 419 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 420 Should Contain Any ${pel_records['${id}']['Message']} internal failure ignore_case=True 421 Should Contain Any ${elog_entry[0]['Message']} InternalFailure ignore_case=True 422 423 Should Be Equal ${redfish_log_time} ${pel_log_time} 424 425 426Verify Delete All PEL 427 [Documentation] Verify deleting all PEL logs. 428 [Tags] Verify_Delete_All_PEL 429 430 Create Test PEL Log 431 Create Test PEL Log 432 Peltool --delete-all False 433 434 ${pel_ids}= Get PEL Log Via BMC CLI 435 Should Be Empty ${pel_ids} 436 437 438Verify Informational Error Log 439 [Documentation] Create an informational error log and verify. 440 [Tags] Verify_Informational_Error_Log 441 442 Redfish Purge Event Log 443 # Create an informational error log. 444 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 445 ${pel_records}= Peltool -lfh 446 447 # An example of information error log data: 448 # { 449 # "0x500006A0": { 450 # "SRC": "BD8D1002", 451 # "Message": "An application had an internal failure", 452 # "PLID": "0x500006A0", 453 # "CreatorID": "BMC", 454 # "Subsystem": "BMC Firmware", 455 # "Commit Time": "10/14/2020 11:41:38", 456 # "Sev": "Informational Event", 457 # "CompID": "0x1000" 458 # } 459 # } 460 461 ${ids}= Get Dictionary Keys ${pel_records} 462 ${id}= Get From List ${ids} 0 463 Should Contain ${pel_records['${id}']['Sev']} Informational 464 465 466Verify Predictable Error Log 467 [Documentation] Create a predictive error and verify. 468 [Tags] Verify_Predictable_Error_Log 469 470 # Create a predictable error log. 471 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 472 ${pel_records}= Peltool -l 473 474 # An example of predictive error log data: 475 # { 476 # "0x5000069E": { 477 # "SRC": "BD8D1002", 478 # "Message": "An application had an internal failure", 479 # "PLID": "0x5000069E", 480 # "CreatorID": "BMC", 481 # "Subsystem": "BMC Firmware", 482 # "Commit Time": "10/14/2020 11:40:07", 483 # "Sev": "Predictive Error", 484 # "CompID": "0x1000" 485 # } 486 # } 487 488 ${pel_ids}= Get PEL Log Via BMC CLI 489 ${id}= Get From List ${pel_ids} -1 490 Should Contain ${pel_records['${id}']['Sev']} Predictive 491 492 493Verify Unrecoverable Error Log 494 [Documentation] Create an unrecoverable error and verify. 495 [Tags] Verify_Unrecoverable_Error_Log 496 497 # Create an internal failure error log. 498 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 499 ${pel_records}= Peltool -l 500 501 # An example of unrecoverable error log data: 502 # { 503 # "0x50000CC5": { 504 # "SRC": "BD8D1002", 505 # "Message": "An application had an internal failure", 506 # "PLID": "0x50000CC5", 507 # "CreatorID": "BMC", 508 # "Subsystem": "BMC Firmware", 509 # "Commit Time": "04/01/2020 16:44:55", 510 # "Sev": "Unrecoverable Error", 511 # "CompID": "0x1000" 512 # } 513 # } 514 515 ${pel_ids}= Get PEL Log Via BMC CLI 516 ${id}= Get From List ${pel_ids} -1 517 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 518 519 520Verify Error Logging Rotation Policy 521 [Documentation] Verify error logging rotation policy. 522 [Tags] Verify_Error_Logging_Rotation_Policy 523 [Template] Error Logging Rotation Policy 524 525 # Error log type Max allocated space % of total logging space 526 Informational 15 527 Unrecoverable 30 528 Predictive 30 529 530Verify Reverse Order Of PEL Logs 531 [Documentation] Verify PEL command to output PEL logs in reverse order. 532 [Tags] Verify_Reverse_PEL_Logs 533 534 Redfish Purge Event Log 535 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 536 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 537 538 ${pel_records}= Peltool -rl 539 ${pel_ids}= Get Dictionary Keys ${pel_records} False 540 541 Should Be True ${pel_ids}[0] > ${pel_ids}[1] 542 543 544Verify Total PEL Count 545 [Documentation] Verify total PEL count returned by peltool command. 546 [Tags] Verify_Total_PEL_Count 547 548 # Initially remove all logs. 549 Redfish Purge Event Log 550 551 # Generate a random number between 1-20. 552 ${random}= Evaluate random.randint(1, 20) modules=random 553 554 # Generate predictive error log multiple times. 555 FOR ${count} IN RANGE 0 ${random} 556 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 557 END 558 559 # Check PEL log count via peltool command and compare it with actual generated log count. 560 ${pel_records}= peltool -n 561 562 Should Be Equal ${pel_records['Number of PELs found']} ${random} 563 564 565Verify Listing Information Error 566 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 567 [Tags] Verify_Listing_Information_Error 568 569 # Initially remove all logs. 570 Redfish Purge Event Log 571 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 572 573 # Generate informational logs and verify that it would not get listed by peltool's list command. 574 ${pel_records}= peltool -l 575 ${pel_ids}= Get PEL Log Via BMC CLI 576 ${id}= Get From List ${pel_ids} -1 577 Should Not Contain ${pel_records['${id}']['Sev']} Informational 578 579 # Verify that information logs get listed using peltool's list command with -lfh option. 580 ${pel_records}= peltool -lfh 581 ${pel_ids}= Get PEL Log Via BMC CLI 582 ${id}= Get From List ${pel_ids} -1 583 Should Contain ${pel_records['${id}']['Sev']} Informational 584 585 586Verify PEL Delete 587 [Documentation] Verify that peltool command can delete PEL log based on id. 588 [Tags] Verify_PEL_Delete 589 590 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 591 ${pel_ids}= Get PEL Log Via BMC CLI 592 ${id}= Get From List ${pel_ids} -1 593 Peltool -d ${id} 594 ${output}= peltool -i ${id} 595 Should Contain ${output} PEL not found 596 597 598*** Keywords *** 599 600Error Logging Rotation Policy 601 [Documentation] Verify that when maximum log limit is reached, given error logging type 602 ... are deleted when reached their max allocated space. 603 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 604 605 # Description of argument(s): 606 # error_log_type Error log type. 607 # max_allocated_space_percentage The maximum percentage of disk usage for given error 608 # log type when maximum count/log size is reached. 609 # The maximum error log count is 3000. 610 611 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 612 # in BMC during the test. 613 Redfish Purge Event Log 614 615 # Determine the log generating command as per type of error. 616 ${cmd}= Set Variable If 617 ... '${error_log_type}' == 'Informational' ${CMD_INFORMATIONAL_ERROR} 618 ... '${error_log_type}' == 'Unrecoverable' ${CMD_UNRECOVERABLE_ERROR} 619 ... '${error_log_type}' == 'Predictive' ${CMD_PREDICTIVE_ERROR} 620 621 # Create 3001 information logs. The logging disk capacity limit is set to 20MB and the max log 622 # count limit is 3000. Once log count crosses the limit, both BMC and non BMC created information, 623 # non-informational logs are reduced to 15% and 30% respectively(i.e. 3 MB, 6 MB). 624 625 FOR ${count} IN RANGE 0 3001 626 BMC Execute Command ${cmd} 627 END 628 629 # Delay for BMC to perform delete older error logs when log limit exceeds. 630 Sleep 10s 631 632 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 633 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 634 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 635 636 ${disk_usage_percentage}= Get Disk Usage For Error Logs 637 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 638 ${percent_diff}= Evaluate abs(${percent_diff}) 639 Should Be True ${percent_diff} <= 0.5 640 641 642Get Disk Usage For Error Logs 643 [Documentation] Get disk usage percentage for error logs. 644 645 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 646 647 ${usage_output}= Fetch From Left ${usage_output} \/ 648 649 # Covert disk usage unit from KB to MB. 650 ${usage_output}= Evaluate ${usage_output} / 1024 651 652 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 653 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 654 655 [return] ${usage_percent} 656 657 658Create Test PEL Log 659 [Documentation] Generate test PEL log. 660 [Arguments] ${pel_type}=Internal Failure 661 662 # Description of argument(s): 663 # pel_type The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout). 664 665 # Test PEL log entry example: 666 # { 667 # "0x5000002D": { 668 # "SRC": "BD8D1002", 669 # "Message": "An application had an internal failure", 670 # "PLID": "0x5000002D", 671 # "CreatorID": "BMC", 672 # "Subsystem": "BMC Firmware", 673 # "Commit Time": "02/25/2020 04:47:09", 674 # "Sev": "Unrecoverable Error", 675 # "CompID": "0x1000" 676 # } 677 # } 678 679 Run Keyword If '${pel_type}' == 'Internal Failure' 680 ... BMC Execute Command ${CMD_INTERNAL_FAILURE} 681 ... ELSE IF '${pel_type}' == 'FRU Callout' 682 ... BMC Execute Command ${CMD_FRU_CALLOUT} 683 ... ELSE IF '${pel_type}' == 'Procedure And Symbolic FRU Callout' 684 ... BMC Execute Command ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} 685 686 687Get PEL Log IDs 688 [Documentation] Returns the list of PEL log IDs which contains given field's value. 689 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 690 691 # Description of argument(s): 692 # pel_section The section of PEL (e.g. Private Header, User Header). 693 # pel_field The PEL field (e.g. Event Severity, Event Type). 694 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 695 696 ${pel_ids}= Get PEL Log Via BMC CLI 697 @{pel_id_list}= Create List 698 699 FOR ${id} IN @{pel_ids} 700 ${pel_output}= Peltool -i ${id} 701 # Example of PEL output from "peltool -i <id>" command. 702 # [Private Header]: 703 # [Created at]: 08/24/1928 12:04:06 704 # [Created by]: 0x584D 705 # [Sub-section type]: 0 706 # [Entry Id]: 0x50000BB7 707 # [Platform Log Id]: 0x8200061D 708 # [CSSVER]: 709 # [Section Version]: 1 710 # [Creator Subsystem]: PHYP 711 # [BMC Event Log Id]: 341 712 # [Committed at]: 03/25/1920 12:06:22 713 # [User Header]: 714 # [Log Committed by]: 0x4552 715 # [Action Flags]: 716 # [0]: Report Externally 717 # [Subsystem]: I/O Subsystem 718 # [Event Type]: Miscellaneous, Informational Only 719 # [Sub-section type]: 0 720 # [Event Scope]: Entire Platform 721 # [Event Severity]: Informational Event 722 # [Host Transmission]: Not Sent 723 # [Section Version]: 1 724 725 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 726 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 727 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 728 END 729 Sort List ${pel_id_list} 730 731 [Return] ${pel_id_list} 732 733 734Get PEL Log Via BMC CLI 735 [Documentation] Returns the list of PEL IDs using BMC CLI. 736 737 ${pel_records}= Peltool -l 738 ${ids}= Get Dictionary Keys ${pel_records} 739 Sort List ${ids} 740 741 [Return] ${ids} 742 743 744Get PEL Field Value 745 [Documentation] Returns the value of given PEL's field. 746 [Arguments] ${pel_id} ${pel_section} ${pel_field} 747 748 # Description of argument(s): 749 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 750 # pel_section The section of PEL (e.g. Private Header, User Header) 751 # pel_field The PEL field (e.g. Event Severity, Event Type). 752 753 ${pel_output}= Peltool -i ${pel_id} 754 755 # Example of PEL output from "peltool -i <id>" command. 756 # [Private Header]: 757 # [Created at]: 08/24/1928 12:04:06 758 # [Created by]: 0x584D 759 # [Sub-section type]: 0 760 # [Entry Id]: 0x50000BB7 761 # [Platform Log Id]: 0x8200061D 762 # [CSSVER]: 763 # [Section Version]: 1 764 # [Creator Subsystem]: PHYP 765 # [BMC Event Log Id]: 341 766 # [Committed at]: 03/25/1920 12:06:22 767 # [User Header]: 768 # [Log Committed by]: 0x4552 769 # [Action Flags]: 770 # [0]: Report Externally 771 # [Subsystem]: I/O Subsystem 772 # [Event Type]: Miscellaneous, Informational Only 773 # [Sub-section type]: 0 774 # [Event Scope]: Entire Platform 775 # [Event Severity]: Informational Event 776 # [Host Transmission]: Not Sent 777 # [Section Version]: 1 778 779 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 780 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 781 782 [Return] ${pel_field_output} 783