1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/openbmc_ffdc.robot 9 10Test Setup Redfish.Login 11Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 12 13 14*** Variables *** 15 16${CMD_INTERNAL_FAILURE} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 17... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 18... xyz.openbmc_project.Logging.Entry.Level.Error 0 19 20${CMD_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 21... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.Timeout 22... xyz.openbmc_project.Logging.Entry.Level.Error 2 "TIMEOUT_IN_MSEC" "5" 23... "CALLOUT_INVENTORY_PATH" "/xyz/openbmc_project/inventory/system/chassis/motherboard" 24 25${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 26... xyz.openbmc_project.Logging.Create Create ssa{ss} org.open_power.Logging.Error.TestError1 27... xyz.openbmc_project.Logging.Entry.Level.Error 0 28 29${CMD_INFORMATIONAL_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 30... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.TestError2 31... xyz.openbmc_project.Logging.Entry.Level.Informational 0 32 33${CMD_INVENTORY_PREFIX} busctl get-property xyz.openbmc_project.Inventory.Manager 34... /xyz/openbmc_project/inventory/system/chassis/motherboard 35 36${CMD_UNRECOVERABLE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 37... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 38... xyz.openbmc_project.Logging.Entry.Level.Error 0 39 40${CMD_PREDICTIVE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 41... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 42... xyz.openbmc_project.Logging.Entry.Level.Warning 0 43 44@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 45 46*** Test Cases *** 47 48Create Test PEL Log And Verify 49 [Documentation] Create PEL log using busctl command and verify via peltool. 50 [Tags] Create_Test_PEL_Log_And_Verify 51 52 Redfish Purge Event Log 53 Create Test PEL Log 54 ${pel_id}= Get PEL Log Via BMC CLI 55 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 56 57 58Verify PEL Log Details 59 [Documentation] Verify PEL log details via peltool. 60 [Tags] Verify_PEL_Log_Details 61 62 Redfish Purge Event Log 63 64 ${bmc_time1}= CLI Get BMC DateTime 65 Create Test PEL Log 66 ${bmc_time2}= CLI Get BMC DateTime 67 68 ${pel_records}= Peltool -l 69 70 # Example output from 'Peltool -l': 71 # pel_records: 72 # [0x50000012]: 73 # [CreatorID]: BMC 74 # [CompID]: 0x1000 75 # [PLID]: 0x50000012 76 # [Subsystem]: BMC Firmware 77 # [Message]: An application had an internal failure 78 # [SRC]: BD8D1002 79 # [Commit Time]: 03/02/2020 09:35:15 80 # [Sev]: Unrecoverable Error 81 82 ${ids}= Get Dictionary Keys ${pel_records} 83 ${id}= Get From List ${ids} 0 84 85 @{pel_fields}= Create List CreatorID Subsystem Message Sev 86 FOR ${field} IN @{pel_fields} 87 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 88 END 89 90 Valid Value pel_records['${id}']['PLID'] ['${id}'] 91 92 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 93 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 94 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 95 96 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 97 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 98 99 # Convert BMC and PEL time to epoch time before comparing. 100 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 101 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 102 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 103 104 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 105 106 107Verify Mandatory Sections Of Error Log PEL 108 [Documentation] Verify mandatory sections of error log PEL. 109 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 110 111 Create Test PEL Log 112 113 ${pel_ids}= Get PEL Log Via BMC CLI 114 ${pel_id}= Get From List ${pel_ids} -1 115 ${pel_output}= Peltool -i ${pel_id} 116 ${pel_sections}= Get Dictionary Keys ${pel_output} 117 118 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 119 120 121Verify PEL Log Persistence After BMC Reboot 122 [Documentation] Verify PEL log persistence after BMC reboot. 123 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 124 125 Create Test PEL Log 126 ${pel_before_reboot}= Get PEL Log Via BMC CLI 127 128 Redfish OBMC Reboot (off) 129 ${pel_after_reboot}= Get PEL Log Via BMC CLI 130 131 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 132 133 134Verify PEL ID Numbering 135 [Documentation] Verify PEL ID numbering. 136 [Tags] Verify_PEL_ID_Numbering 137 138 Redfish Purge Event Log 139 Create Test PEL Log 140 Create Test PEL Log 141 142 ${pel_ids}= Get PEL Log Via BMC CLI 143 144 # Example of PEL IDs from PEL logs. 145 # [0x50000012]: <--- First PEL ID 146 # [CreatorID]: BMC 147 # [CompID]: 0x1000 148 # [PLID]: 0x50000012 149 # [Subsystem]: BMC Firmware 150 # [Message]: An application had an internal failure 151 # [SRC]: BD8D1002 152 # [Commit Time]: 03/02/2020 09:35:15 153 # [Sev]: Unrecoverable Error 154 # 155 # [0x50000013]: <--- Second PEL ID 156 # [CreatorID]: BMC 157 # [CompID]: 0x1000 158 # [PLID]: 0x50000013 159 # [Subsystem]: BMC Firmware 160 # [Message]: An application had an internal failure 161 # [SRC]: BD8D1002 162 # [Commit Time]: 03/02/2020 09:35:15 163 # [Sev]: Unrecoverable Error 164 165 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 166 167Verify Machine Type Model And Serial Number 168 [Documentation] Verify machine type model and serial number from PEL. 169 [Tags] Verify_Machine_Type_Model_And_Serial_Number 170 171 Create Test PEL Log 172 173 ${pel_ids}= Get PEL Log Via BMC CLI 174 ${id}= Get From List ${pel_ids} -1 175 176 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 177 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 178 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 179 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 180 181 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 182 # [Failing MTMS]: 183 # [Created by]: 0x2000 184 # [Machine Type Model]: 1234-ABC <---- Machine type 185 # [Section Version]: 1 186 # [Serial Number]: ABCDEFG <---- Serial number 187 # [Sub-section type]: 0 188 189 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 190 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 191 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 192 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 193 194 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 195 Valid Value pel_serial_number ['${redfish_serial_number}'] 196 197 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 198 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 199 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 200 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 201 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 202 203 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 204 Valid Value pel_serial_number ['${redfish_serial_number}'] 205 206 207Verify Host Off State From PEL 208 [Documentation] Verify Host off state from PEL. 209 [Tags] Verify_Host_Off_State_From_PEL 210 211 Redfish Power Off stack_mode=skip 212 Create Test PEL Log 213 214 ${pel_ids}= Get PEL Log Via BMC CLI 215 ${id}= Get From List ${pel_ids} -1 216 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 217 218 Valid Value pel_host_state ['Off'] 219 220 221Verify BMC Version From PEL 222 [Documentation] Verify BMC Version from PEL. 223 [Tags] Verify_BMC_Version_From_PEL 224 225 Create Test PEL Log 226 227 ${pel_ids}= Get PEL Log Via BMC CLI 228 ${id}= Get From List ${pel_ids} -1 229 ${pel_bmc_version}= Get PEL Field Value ${id} User Data BMC Version ID 230 231 ${bmc_version}= Get BMC Version 232 Valid Value bmc_version ['${bmc_version}'] 233 234 235Verify PEL Log After Host Poweron 236 [Documentation] Verify PEL log generation while booting host. 237 [Tags] Verify_PEL_Log_After_Host_Poweron 238 239 Redfish Power Off stack_mode=skip 240 Redfish Purge Event Log 241 Redfish Power On stack_mode=skip 242 243 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 244 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 245 246 # Get BMC created non-infomational error. 247 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 248 249 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 250 251 252Verify BMC Event Log ID 253 [Documentation] Verify BMC Event Log ID from PEL. 254 [Tags] Verify_BMC_Event_Log_ID 255 256 Redfish Purge Event Log 257 Create Test PEL Log 258 259 ${pel_ids}= Get PEL Log Via BMC CLI 260 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 261 262 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 263 # [Private Header]: 264 # [Created at]: 08/24/1928 12:04:06 265 # [Created by]: 0x584D 266 # [Sub-section type]: 0 267 # [Entry Id]: 0x50000BB7 268 # [Platform Log Id]: 0x8200061D 269 # [CSSVER]: 270 # [Section Version]: 1 271 # [Creator Subsystem]: PHYP 272 # [BMC Event Log Id]: 341 <---- BMC event log id value 273 # [Committed at]: 03/25/1920 12:06:22 274 275 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 276 277 # Example of redfish_event_logs output: 278 # redfish_event_logs: 279 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 280 # [Name]: System Event Log Entries 281 # [Members@odata.count]: 1 282 # [@odata.type]: #LogEntryCollection.LogEntryCollection 283 # [Description]: Collection of System Event Log Entries 284 # [Members]: 285 # [0]: 286 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 287 # [Name]: System Event Log Entry 288 # [Severity]: Critical 289 # [EntryType]: Event 290 # [Created]: 2020-04-02T07:25:13+00:00 291 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 292 # [Id]: 235 <----- Event log ID 293 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 294 295 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 296 297 298Verify FRU Callout 299 [Documentation] Verify FRU callout entries from PEL log. 300 [Tags] Verify_FRU_Callout 301 302 Create Test PEL Log FRU Callout 303 304 ${pel_ids}= Get PEL Log Via BMC CLI 305 ${id}= Get From List ${pel_ids} -1 306 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 307 308 # Example of PEL Callout Section from "peltool -i <id>" command. 309 # [Callouts]: 310 # [0]: 311 # [FRU Type]: Normal Hardware FRU 312 # [Priority]: Mandatory, replace all with this type as a unit 313 # [Location Code]: U78DA.ND1.1234567-P0 314 # [Part Number]: F191014 315 # [CCIN]: 2E2D 316 # [Serial Number]: YL2E2D010000 317 # [Callout Count]: 1 318 319 Valid Value pel_callout_section['Callout Count'] ['1'] 320 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 321 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 322 323 # Verify Location Code field of PEL callout with motherboard's Location Code. 324 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 325 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 326 ... ${busctl_output[0].split('"')[1].strip('"')} 327 328 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 329 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 330 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 331 332 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 333 ${busctl_output}= BMC Execute Command 334 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 335 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 336 ... ${busctl_output[0].split('"')[1].strip('"')} 337 338 339Verify Procedure And Symbolic FRU Callout 340 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 341 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 342 343 Create Test PEL Log Procedure And Symbolic FRU Callout 344 345 ${pel_ids}= Get PEL Log Via BMC CLI 346 ${id}= Get From List ${pel_ids} -1 347 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 348 349 # Example of PEL Callout Section from "peltool -i <id>" command. 350 # [Callouts]: 351 # [0]: 352 # [Priority]: Mandatory, replace all with this type as a unit 353 # [Procedure Number]: BMCSP02 354 # [FRU Type]: Maintenance Procedure Required 355 # [1]: 356 # [Priority]: Medium Priority 357 # [Part Number]: SVCDOCS 358 # [FRU Type]: Symbolic FRU 359 # [Callout Count]: 2 360 361 Valid Value pel_callout_section['Callout Count'] ['2'] 362 363 # Verify procedural callout info. 364 365 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 366 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 367 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 368 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 369 370 # Verify procedural callout info. 371 372 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 373 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 374 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 375 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 376 377 378Verify PEL Log Entry For Event Log 379 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 380 [Tags] Verify_PEL_Log_Entry_For_Event_Log 381 382 Redfish Purge Event Log 383 # Create an internal failure error log. 384 BMC Execute Command ${CMD_INTERNAL_FAILURE} 385 386 ${elog_entry}= Get Event Logs 387 # Example of Redfish event logs: 388 # elog_entry: 389 # [0]: 390 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 391 # [Created]: 2020-04-20T01:55:22+00:00 392 # [Id]: 1 393 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 394 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 395 # [EntryType]: Event 396 # [Severity]: Critical 397 # [Name]: System Event Log Entry 398 399 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 400 401 ${pel_records}= Peltool -l 402 # Example output from 'Peltool -l': 403 # pel_records: 404 # [0x50000023]: 405 # [SRC]: BD8D1002 406 # [CreatorID]: BMC 407 # [Message]: An application had an internal failure 408 # [CompID]: 0x1000 409 # [PLID]: 0x50000023 410 # [Commit Time]: 04/20/2020 01:55:22 411 # [Subsystem]: BMC Firmware 412 # [Sev]: Unrecoverable Error 413 414 ${ids}= Get Dictionary Keys ${pel_records} 415 ${id}= Get From List ${ids} 0 416 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 417 ... date_format=%m/%d/%Y %H:%M:%S 418 419 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 420 Should Contain Any ${pel_records['${id}']['Message']} internal failure ignore_case=True 421 Should Contain Any ${elog_entry[0]['Message']} InternalFailure ignore_case=True 422 423 Should Be Equal ${redfish_log_time} ${pel_log_time} 424 425 426Verify Delete All PEL 427 [Documentation] Verify deleting all PEL logs. 428 [Tags] Verify_Delete_All_PEL 429 430 Create Test PEL Log 431 Create Test PEL Log 432 Peltool --delete-all False 433 434 ${pel_ids}= Get PEL Log Via BMC CLI 435 Should Be Empty ${pel_ids} 436 437 438Verify Informational Error Log 439 [Documentation] Create an informational error log and verify. 440 [Tags] Verify_Informational_Error_Log 441 442 Redfish Purge Event Log 443 # Create an informational error log. 444 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 445 ${pel_records}= Peltool -lfh 446 447 # An example of information error log data: 448 # { 449 # "0x500006A0": { 450 # "SRC": "BD8D1002", 451 # "Message": "An application had an internal failure", 452 # "PLID": "0x500006A0", 453 # "CreatorID": "BMC", 454 # "Subsystem": "BMC Firmware", 455 # "Commit Time": "10/14/2020 11:41:38", 456 # "Sev": "Informational Event", 457 # "CompID": "0x1000" 458 # } 459 # } 460 461 ${ids}= Get Dictionary Keys ${pel_records} 462 ${id}= Get From List ${ids} 0 463 Should Contain ${pel_records['${id}']['Sev']} Informational 464 465 466Verify Predictable Error Log 467 [Documentation] Create a predictive error and verify. 468 [Tags] Verify_Predictable_Error_Log 469 470 # Create a predictable error log. 471 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 472 ${pel_records}= Peltool -l 473 474 # An example of predictive error log data: 475 # { 476 # "0x5000069E": { 477 # "SRC": "BD8D1002", 478 # "Message": "An application had an internal failure", 479 # "PLID": "0x5000069E", 480 # "CreatorID": "BMC", 481 # "Subsystem": "BMC Firmware", 482 # "Commit Time": "10/14/2020 11:40:07", 483 # "Sev": "Predictive Error", 484 # "CompID": "0x1000" 485 # } 486 # } 487 488 ${pel_ids}= Get PEL Log Via BMC CLI 489 ${id}= Get From List ${pel_ids} -1 490 Should Contain ${pel_records['${id}']['Sev']} Predictive 491 492 493Verify Unrecoverable Error Log 494 [Documentation] Create an unrecoverable error and verify. 495 [Tags] Verify_Unrecoverable_Error_Log 496 497 # Create an internal failure error log. 498 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 499 ${pel_records}= Peltool -l 500 501 # An example of unrecoverable error log data: 502 # { 503 # "0x50000CC5": { 504 # "SRC": "BD8D1002", 505 # "Message": "An application had an internal failure", 506 # "PLID": "0x50000CC5", 507 # "CreatorID": "BMC", 508 # "Subsystem": "BMC Firmware", 509 # "Commit Time": "04/01/2020 16:44:55", 510 # "Sev": "Unrecoverable Error", 511 # "CompID": "0x1000" 512 # } 513 # } 514 515 ${pel_ids}= Get PEL Log Via BMC CLI 516 ${id}= Get From List ${pel_ids} -1 517 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 518 519 520Verify Error Logging Rotation Policy 521 [Documentation] Verify error logging rotation policy. 522 [Tags] Verify_Error_Logging_Rotation_Policy 523 [Template] Error Logging Rotation Policy 524 525 # Error log type Max allocated space % of total logging space 526 Informational 15 527 Unrecoverable 30 528 Predictive 30 529 530 531Verify Reverse Order Of PEL Logs 532 [Documentation] Verify PEL command to output PEL logs in reverse order. 533 [Tags] Verify_Reverse_PEL_Logs 534 535 Redfish Purge Event Log 536 537 # Below commands create unrecoverable error log at first and then the predictable error. 538 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 539 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 540 541 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 542 ${pel_records}= peltool -lr 543 544 # It is found that, variables like dictionary always keep items in sorted order that makes 545 # this verification not possible, hence json is used to keep the items original order. 546 ${pel_records}= Convert To String ${pel_records} 547 ${json_string}= Replace String ${pel_records} ' " 548 ${json_object}= Evaluate json.loads('''${json_string}''') json 549 550 ${list}= Convert To List ${json_object} 551 552 ${id1}= Get From List ${list} 0 553 ${id1}= Convert To Integer ${id1} 554 ${id2}= Get From List ${list} 1 555 ${id2}= Convert To Integer ${id2} 556 557 Should Be True ${id1} > ${id2} 558 559 560Verify Total PEL Count 561 [Documentation] Verify total PEL count returned by peltool command. 562 [Tags] Verify_Total_PEL_Count 563 564 # Initially remove all logs. 565 Redfish Purge Event Log 566 567 # Generate a random number between 1-20. 568 ${random}= Evaluate random.randint(1, 20) modules=random 569 570 # Generate predictive error log multiple times. 571 FOR ${count} IN RANGE 0 ${random} 572 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 573 END 574 575 # Check PEL log count via peltool command and compare it with actual generated log count. 576 ${pel_records}= peltool -n 577 578 Should Be Equal ${pel_records['Number of PELs found']} ${random} 579 580 581Verify Listing Information Error 582 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 583 [Tags] Verify_Listing_Information_Error 584 585 # Initially remove all logs. 586 Redfish Purge Event Log 587 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 588 589 # Generate informational logs and verify that it would not get listed by peltool's list command. 590 ${pel_records}= peltool -l 591 ${ids}= Get Dictionary Keys ${pel_records} 592 Should Be Empty ${ids} 593 594 # Verify that information logs get listed using peltool's list command with -lfh option. 595 ${pel_records}= peltool -lfh 596 ${ids}= Get Dictionary Keys ${pel_records} 597 Should Not Be Empty ${ids} 598 ${id}= Get From List ${ids} 0 599 Should Contain ${pel_records['${id}']['Sev']} Informational 600 601 602Verify PEL Delete 603 [Documentation] Verify that peltool command can delete PEL log based on id. 604 [Tags] Verify_PEL_Delete 605 606 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 607 ${pel_ids}= Get PEL Log Via BMC CLI 608 ${id}= Get From List ${pel_ids} -1 609 Peltool -d ${id} 610 ${output}= peltool -i ${id} 611 Should Contain ${output} PEL not found 612 613 614*** Keywords *** 615 616Error Logging Rotation Policy 617 [Documentation] Verify that when maximum log limit is reached, given error logging type 618 ... are deleted when reached their max allocated space. 619 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 620 621 # Description of argument(s): 622 # error_log_type Error log type. 623 # max_allocated_space_percentage The maximum percentage of disk usage for given error 624 # log type when maximum count/log size is reached. 625 # The maximum error log count is 3000. 626 627 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 628 # in BMC during the test. 629 Redfish Purge Event Log 630 631 # Determine the log generating command as per type of error. 632 ${cmd}= Set Variable If 633 ... '${error_log_type}' == 'Informational' ${CMD_INFORMATIONAL_ERROR} 634 ... '${error_log_type}' == 'Unrecoverable' ${CMD_UNRECOVERABLE_ERROR} 635 ... '${error_log_type}' == 'Predictive' ${CMD_PREDICTIVE_ERROR} 636 637 # Create 3001 information logs. The logging disk capacity limit is set to 20MB and the max log 638 # count limit is 3000. Once log count crosses the limit, both BMC and non BMC created information, 639 # non-informational logs are reduced to 15% and 30% respectively(i.e. 3 MB, 6 MB). 640 641 FOR ${count} IN RANGE 0 3001 642 BMC Execute Command ${cmd} 643 END 644 645 # Delay for BMC to perform delete older error logs when log limit exceeds. 646 Sleep 10s 647 648 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 649 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 650 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 651 652 ${disk_usage_percentage}= Get Disk Usage For Error Logs 653 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 654 ${percent_diff}= Evaluate abs(${percent_diff}) 655 Should Be True ${percent_diff} <= 0.5 656 657 658Get Disk Usage For Error Logs 659 [Documentation] Get disk usage percentage for error logs. 660 661 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 662 663 ${usage_output}= Fetch From Left ${usage_output} \/ 664 665 # Covert disk usage unit from KB to MB. 666 ${usage_output}= Evaluate ${usage_output} / 1024 667 668 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 669 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 670 671 [return] ${usage_percent} 672 673 674Create Test PEL Log 675 [Documentation] Generate test PEL log. 676 [Arguments] ${pel_type}=Internal Failure 677 678 # Description of argument(s): 679 # pel_type The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout). 680 681 # Test PEL log entry example: 682 # { 683 # "0x5000002D": { 684 # "SRC": "BD8D1002", 685 # "Message": "An application had an internal failure", 686 # "PLID": "0x5000002D", 687 # "CreatorID": "BMC", 688 # "Subsystem": "BMC Firmware", 689 # "Commit Time": "02/25/2020 04:47:09", 690 # "Sev": "Unrecoverable Error", 691 # "CompID": "0x1000" 692 # } 693 # } 694 695 Run Keyword If '${pel_type}' == 'Internal Failure' 696 ... BMC Execute Command ${CMD_INTERNAL_FAILURE} 697 ... ELSE IF '${pel_type}' == 'FRU Callout' 698 ... BMC Execute Command ${CMD_FRU_CALLOUT} 699 ... ELSE IF '${pel_type}' == 'Procedure And Symbolic FRU Callout' 700 ... BMC Execute Command ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} 701 702 703Get PEL Log IDs 704 [Documentation] Returns the list of PEL log IDs which contains given field's value. 705 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 706 707 # Description of argument(s): 708 # pel_section The section of PEL (e.g. Private Header, User Header). 709 # pel_field The PEL field (e.g. Event Severity, Event Type). 710 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 711 712 ${pel_ids}= Get PEL Log Via BMC CLI 713 @{pel_id_list}= Create List 714 715 FOR ${id} IN @{pel_ids} 716 ${pel_output}= Peltool -i ${id} 717 # Example of PEL output from "peltool -i <id>" command. 718 # [Private Header]: 719 # [Created at]: 08/24/1928 12:04:06 720 # [Created by]: 0x584D 721 # [Sub-section type]: 0 722 # [Entry Id]: 0x50000BB7 723 # [Platform Log Id]: 0x8200061D 724 # [CSSVER]: 725 # [Section Version]: 1 726 # [Creator Subsystem]: PHYP 727 # [BMC Event Log Id]: 341 728 # [Committed at]: 03/25/1920 12:06:22 729 # [User Header]: 730 # [Log Committed by]: 0x4552 731 # [Action Flags]: 732 # [0]: Report Externally 733 # [Subsystem]: I/O Subsystem 734 # [Event Type]: Miscellaneous, Informational Only 735 # [Sub-section type]: 0 736 # [Event Scope]: Entire Platform 737 # [Event Severity]: Informational Event 738 # [Host Transmission]: Not Sent 739 # [Section Version]: 1 740 741 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 742 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 743 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 744 END 745 Sort List ${pel_id_list} 746 747 [Return] ${pel_id_list} 748 749 750Get PEL Log Via BMC CLI 751 [Documentation] Returns the list of PEL IDs using BMC CLI. 752 753 ${pel_records}= Peltool -l 754 ${ids}= Get Dictionary Keys ${pel_records} 755 Sort List ${ids} 756 757 [Return] ${ids} 758 759 760Get PEL Field Value 761 [Documentation] Returns the value of given PEL's field. 762 [Arguments] ${pel_id} ${pel_section} ${pel_field} 763 764 # Description of argument(s): 765 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 766 # pel_section The section of PEL (e.g. Private Header, User Header) 767 # pel_field The PEL field (e.g. Event Severity, Event Type). 768 769 ${pel_output}= Peltool -i ${pel_id} 770 771 # Example of PEL output from "peltool -i <id>" command. 772 # [Private Header]: 773 # [Created at]: 08/24/1928 12:04:06 774 # [Created by]: 0x584D 775 # [Sub-section type]: 0 776 # [Entry Id]: 0x50000BB7 777 # [Platform Log Id]: 0x8200061D 778 # [CSSVER]: 779 # [Section Version]: 1 780 # [Creator Subsystem]: PHYP 781 # [BMC Event Log Id]: 341 782 # [Committed at]: 03/25/1920 12:06:22 783 # [User Header]: 784 # [Log Committed by]: 0x4552 785 # [Action Flags]: 786 # [0]: Report Externally 787 # [Subsystem]: I/O Subsystem 788 # [Event Type]: Miscellaneous, Informational Only 789 # [Sub-section type]: 0 790 # [Event Scope]: Entire Platform 791 # [Event Severity]: Informational Event 792 # [Host Transmission]: Not Sent 793 # [Section Version]: 1 794 795 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 796 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 797 798 [Return] ${pel_field_output} 799