1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17${CMD_INTERNAL_FAILURE} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 18... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 19... xyz.openbmc_project.Logging.Entry.Level.Error 0 20 21${CMD_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 22... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.Timeout 23... xyz.openbmc_project.Logging.Entry.Level.Error 2 "TIMEOUT_IN_MSEC" "5" 24... "CALLOUT_INVENTORY_PATH" "/xyz/openbmc_project/inventory/system/chassis/motherboard" 25 26${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 27... xyz.openbmc_project.Logging.Create Create ssa{ss} org.open_power.Logging.Error.TestError1 28... xyz.openbmc_project.Logging.Entry.Level.Error 0 29 30${CMD_INFORMATIONAL_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 31... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.TestError2 32... xyz.openbmc_project.Logging.Entry.Level.Informational 0 33 34${CMD_INVENTORY_PREFIX} busctl get-property xyz.openbmc_project.Inventory.Manager 35... /xyz/openbmc_project/inventory/system/chassis/motherboard 36 37${CMD_UNRECOVERABLE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 38... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 39... xyz.openbmc_project.Logging.Entry.Level.Error 0 40 41${CMD_PREDICTIVE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 42... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 43... xyz.openbmc_project.Logging.Entry.Level.Warning 0 44 45${CMD_UNRECOVERABLE_HOST_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 46... xyz.openbmc_project.Logging.Create Create ssa{ss} 47... xyz.openbmc_project.Host.Error.Event xyz.openbmc_project.Logging.Entry.Level.Error 1 48... RAWPEL /tmp/FILE_NBMC_UNRECOVERABLE 49 50@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 51 52*** Test Cases *** 53 54Create Test PEL Log And Verify 55 [Documentation] Create PEL log using busctl command and verify via peltool. 56 [Tags] Create_Test_PEL_Log_And_Verify 57 58 Redfish Purge Event Log 59 Create Test PEL Log 60 ${pel_id}= Get PEL Log Via BMC CLI 61 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 62 63 64Verify PEL Log Details 65 [Documentation] Verify PEL log details via peltool. 66 [Tags] Verify_PEL_Log_Details 67 68 Redfish Purge Event Log 69 70 ${bmc_time1}= CLI Get BMC DateTime 71 Create Test PEL Log 72 ${bmc_time2}= CLI Get BMC DateTime 73 74 ${pel_records}= Peltool -l 75 76 # Example output from 'Peltool -l': 77 # pel_records: 78 # [0x50000012]: 79 # [CreatorID]: BMC 80 # [CompID]: 0x1000 81 # [PLID]: 0x50000012 82 # [Subsystem]: BMC Firmware 83 # [Message]: An application had an internal failure 84 # [SRC]: BD8D1002 85 # [Commit Time]: 03/02/2020 09:35:15 86 # [Sev]: Unrecoverable Error 87 88 ${ids}= Get Dictionary Keys ${pel_records} 89 ${id}= Get From List ${ids} 0 90 91 @{pel_fields}= Create List CreatorID Subsystem Message Sev 92 FOR ${field} IN @{pel_fields} 93 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 94 END 95 96 Valid Value pel_records['${id}']['PLID'] ['${id}'] 97 98 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 99 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 100 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 101 102 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 103 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 104 105 # Convert BMC and PEL time to epoch time before comparing. 106 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 107 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 108 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 109 110 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 111 112 113Verify Mandatory Sections Of Error Log PEL 114 [Documentation] Verify mandatory sections of error log PEL. 115 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 116 117 Create Test PEL Log 118 119 ${pel_ids}= Get PEL Log Via BMC CLI 120 ${pel_id}= Get From List ${pel_ids} -1 121 ${pel_output}= Peltool -i ${pel_id} 122 ${pel_sections}= Get Dictionary Keys ${pel_output} 123 124 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 125 126 127Verify PEL Log Persistence After BMC Reboot 128 [Documentation] Verify PEL log persistence after BMC reboot. 129 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 130 131 Create Test PEL Log 132 ${pel_before_reboot}= Get PEL Log Via BMC CLI 133 134 Redfish OBMC Reboot (off) 135 ${pel_after_reboot}= Get PEL Log Via BMC CLI 136 137 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 138 139 140Verify PEL ID Numbering 141 [Documentation] Verify PEL ID numbering. 142 [Tags] Verify_PEL_ID_Numbering 143 144 Redfish Purge Event Log 145 Create Test PEL Log 146 Create Test PEL Log 147 148 ${pel_ids}= Get PEL Log Via BMC CLI 149 150 # Example of PEL IDs from PEL logs. 151 # [0x50000012]: <--- First PEL ID 152 # [CreatorID]: BMC 153 # [CompID]: 0x1000 154 # [PLID]: 0x50000012 155 # [Subsystem]: BMC Firmware 156 # [Message]: An application had an internal failure 157 # [SRC]: BD8D1002 158 # [Commit Time]: 03/02/2020 09:35:15 159 # [Sev]: Unrecoverable Error 160 # 161 # [0x50000013]: <--- Second PEL ID 162 # [CreatorID]: BMC 163 # [CompID]: 0x1000 164 # [PLID]: 0x50000013 165 # [Subsystem]: BMC Firmware 166 # [Message]: An application had an internal failure 167 # [SRC]: BD8D1002 168 # [Commit Time]: 03/02/2020 09:35:15 169 # [Sev]: Unrecoverable Error 170 171 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 172 173Verify Machine Type Model And Serial Number 174 [Documentation] Verify machine type model and serial number from PEL. 175 [Tags] Verify_Machine_Type_Model_And_Serial_Number 176 177 Create Test PEL Log 178 179 ${pel_ids}= Get PEL Log Via BMC CLI 180 ${id}= Get From List ${pel_ids} -1 181 182 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 183 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 184 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 185 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 186 187 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 188 # [Failing MTMS]: 189 # [Created by]: 0x2000 190 # [Machine Type Model]: 1234-ABC <---- Machine type 191 # [Section Version]: 1 192 # [Serial Number]: ABCDEFG <---- Serial number 193 # [Sub-section type]: 0 194 195 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 196 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 197 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 198 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 199 200 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 201 Valid Value pel_serial_number ['${redfish_serial_number}'] 202 203 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 204 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 205 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 206 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 207 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 208 209 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 210 Valid Value pel_serial_number ['${redfish_serial_number}'] 211 212 213Verify Host Off State From PEL 214 [Documentation] Verify Host off state from PEL. 215 [Tags] Verify_Host_Off_State_From_PEL 216 217 Redfish Power Off stack_mode=skip 218 Create Test PEL Log 219 220 ${pel_ids}= Get PEL Log Via BMC CLI 221 ${id}= Get From List ${pel_ids} -1 222 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 223 224 Valid Value pel_host_state ['Off'] 225 226 227Verify BMC Version From PEL 228 [Documentation] Verify BMC Version from PEL. 229 [Tags] Verify_BMC_Version_From_PEL 230 231 Create Test PEL Log 232 233 ${pel_ids}= Get PEL Log Via BMC CLI 234 ${id}= Get From List ${pel_ids} -1 235 ${pel_bmc_version}= Get PEL Field Value ${id} User Data BMC Version ID 236 237 ${bmc_version}= Get BMC Version 238 Valid Value bmc_version ['${bmc_version}'] 239 240 241Verify PEL Log After Host Poweron 242 [Documentation] Verify PEL log generation while booting host. 243 [Tags] Verify_PEL_Log_After_Host_Poweron 244 245 Redfish Power Off stack_mode=skip 246 Redfish Purge Event Log 247 Redfish Power On stack_mode=skip 248 249 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 250 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 251 252 # Get BMC created non-infomational error. 253 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 254 255 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 256 257 258Verify BMC Event Log ID 259 [Documentation] Verify BMC Event Log ID from PEL. 260 [Tags] Verify_BMC_Event_Log_ID 261 262 Redfish Purge Event Log 263 Create Test PEL Log 264 265 ${pel_ids}= Get PEL Log Via BMC CLI 266 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 267 268 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 269 # [Private Header]: 270 # [Created at]: 08/24/1928 12:04:06 271 # [Created by]: 0x584D 272 # [Sub-section type]: 0 273 # [Entry Id]: 0x50000BB7 274 # [Platform Log Id]: 0x8200061D 275 # [CSSVER]: 276 # [Section Version]: 1 277 # [Creator Subsystem]: PHYP 278 # [BMC Event Log Id]: 341 <---- BMC event log id value 279 # [Committed at]: 03/25/1920 12:06:22 280 281 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 282 283 # Example of redfish_event_logs output: 284 # redfish_event_logs: 285 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 286 # [Name]: System Event Log Entries 287 # [Members@odata.count]: 1 288 # [@odata.type]: #LogEntryCollection.LogEntryCollection 289 # [Description]: Collection of System Event Log Entries 290 # [Members]: 291 # [0]: 292 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 293 # [Name]: System Event Log Entry 294 # [Severity]: Critical 295 # [EntryType]: Event 296 # [Created]: 2020-04-02T07:25:13+00:00 297 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 298 # [Id]: 235 <----- Event log ID 299 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 300 301 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 302 303 304Verify FRU Callout 305 [Documentation] Verify FRU callout entries from PEL log. 306 [Tags] Verify_FRU_Callout 307 308 Create Test PEL Log FRU Callout 309 310 ${pel_ids}= Get PEL Log Via BMC CLI 311 ${id}= Get From List ${pel_ids} -1 312 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 313 314 # Example of PEL Callout Section from "peltool -i <id>" command. 315 # [Callouts]: 316 # [0]: 317 # [FRU Type]: Normal Hardware FRU 318 # [Priority]: Mandatory, replace all with this type as a unit 319 # [Location Code]: U78DA.ND1.1234567-P0 320 # [Part Number]: F191014 321 # [CCIN]: 2E2D 322 # [Serial Number]: YL2E2D010000 323 # [Callout Count]: 1 324 325 Valid Value pel_callout_section['Callout Count'] ['1'] 326 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 327 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 328 329 # Verify Location Code field of PEL callout with motherboard's Location Code. 330 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 331 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 332 ... ${busctl_output[0].split('"')[1].strip('"')} 333 334 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 335 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 336 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 337 338 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 339 ${busctl_output}= BMC Execute Command 340 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 341 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 342 ... ${busctl_output[0].split('"')[1].strip('"')} 343 344 345Verify Procedure And Symbolic FRU Callout 346 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 347 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 348 349 Create Test PEL Log Procedure And Symbolic FRU Callout 350 351 ${pel_ids}= Get PEL Log Via BMC CLI 352 ${id}= Get From List ${pel_ids} -1 353 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 354 355 # Example of PEL Callout Section from "peltool -i <id>" command. 356 # [Callouts]: 357 # [0]: 358 # [Priority]: Mandatory, replace all with this type as a unit 359 # [Procedure Number]: BMCSP02 360 # [FRU Type]: Maintenance Procedure Required 361 # [1]: 362 # [Priority]: Medium Priority 363 # [Part Number]: SVCDOCS 364 # [FRU Type]: Symbolic FRU 365 # [Callout Count]: 2 366 367 Valid Value pel_callout_section['Callout Count'] ['2'] 368 369 # Verify procedural callout info. 370 371 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 372 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 373 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 374 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 375 376 # Verify procedural callout info. 377 378 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 379 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 380 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 381 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 382 383 384Verify PEL Log Entry For Event Log 385 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 386 [Tags] Verify_PEL_Log_Entry_For_Event_Log 387 388 Redfish Purge Event Log 389 # Create an internal failure error log. 390 BMC Execute Command ${CMD_INTERNAL_FAILURE} 391 392 ${elog_entry}= Get Event Logs 393 # Example of Redfish event logs: 394 # elog_entry: 395 # [0]: 396 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 397 # [Created]: 2020-04-20T01:55:22+00:00 398 # [Id]: 1 399 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 400 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 401 # [EntryType]: Event 402 # [Severity]: Critical 403 # [Name]: System Event Log Entry 404 405 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 406 407 ${pel_records}= Peltool -l 408 # Example output from 'Peltool -l': 409 # pel_records: 410 # [0x50000023]: 411 # [SRC]: BD8D1002 412 # [CreatorID]: BMC 413 # [Message]: An application had an internal failure 414 # [CompID]: 0x1000 415 # [PLID]: 0x50000023 416 # [Commit Time]: 04/20/2020 01:55:22 417 # [Subsystem]: BMC Firmware 418 # [Sev]: Unrecoverable Error 419 420 ${ids}= Get Dictionary Keys ${pel_records} 421 ${id}= Get From List ${ids} 0 422 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 423 ... date_format=%m/%d/%Y %H:%M:%S 424 425 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 426 Should Contain Any ${pel_records['${id}']['Message']} internal failure ignore_case=True 427 Should Contain Any ${elog_entry[0]['Message']} InternalFailure ignore_case=True 428 429 Should Be Equal ${redfish_log_time} ${pel_log_time} 430 431 432Verify Delete All PEL 433 [Documentation] Verify deleting all PEL logs. 434 [Tags] Verify_Delete_All_PEL 435 436 Create Test PEL Log 437 Create Test PEL Log 438 Peltool --delete-all False 439 440 ${pel_ids}= Get PEL Log Via BMC CLI 441 Should Be Empty ${pel_ids} 442 443 444Verify Informational Error Log 445 [Documentation] Create an informational error log and verify. 446 [Tags] Verify_Informational_Error_Log 447 448 Redfish Purge Event Log 449 # Create an informational error log. 450 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 451 ${pel_records}= Peltool -lfh 452 453 # An example of information error log data: 454 # { 455 # "0x500006A0": { 456 # "SRC": "BD8D1002", 457 # "Message": "An application had an internal failure", 458 # "PLID": "0x500006A0", 459 # "CreatorID": "BMC", 460 # "Subsystem": "BMC Firmware", 461 # "Commit Time": "10/14/2020 11:41:38", 462 # "Sev": "Informational Event", 463 # "CompID": "0x1000" 464 # } 465 # } 466 467 ${ids}= Get Dictionary Keys ${pel_records} 468 ${id}= Get From List ${ids} 0 469 Should Contain ${pel_records['${id}']['Sev']} Informational 470 471 472Verify Predictable Error Log 473 [Documentation] Create a predictive error and verify. 474 [Tags] Verify_Predictable_Error_Log 475 476 # Create a predictable error log. 477 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 478 ${pel_records}= Peltool -l 479 480 # An example of predictive error log data: 481 # { 482 # "0x5000069E": { 483 # "SRC": "BD8D1002", 484 # "Message": "An application had an internal failure", 485 # "PLID": "0x5000069E", 486 # "CreatorID": "BMC", 487 # "Subsystem": "BMC Firmware", 488 # "Commit Time": "10/14/2020 11:40:07", 489 # "Sev": "Predictive Error", 490 # "CompID": "0x1000" 491 # } 492 # } 493 494 ${pel_ids}= Get PEL Log Via BMC CLI 495 ${id}= Get From List ${pel_ids} -1 496 Should Contain ${pel_records['${id}']['Sev']} Predictive 497 498 499Verify Unrecoverable Error Log 500 [Documentation] Create an unrecoverable error and verify. 501 [Tags] Verify_Unrecoverable_Error_Log 502 503 # Create an internal failure error log. 504 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 505 ${pel_records}= Peltool -l 506 507 # An example of unrecoverable error log data: 508 # { 509 # "0x50000CC5": { 510 # "SRC": "BD8D1002", 511 # "Message": "An application had an internal failure", 512 # "PLID": "0x50000CC5", 513 # "CreatorID": "BMC", 514 # "Subsystem": "BMC Firmware", 515 # "Commit Time": "04/01/2020 16:44:55", 516 # "Sev": "Unrecoverable Error", 517 # "CompID": "0x1000" 518 # } 519 # } 520 521 ${pel_ids}= Get PEL Log Via BMC CLI 522 ${id}= Get From List ${pel_ids} -1 523 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 524 525 526Verify Error Logging Rotation Policy 527 [Documentation] Verify error logging rotation policy. 528 [Tags] Verify_Error_Logging_Rotation_Policy 529 [Template] Error Logging Rotation Policy 530 531 # Error logs to be created % of total logging space when error 532 # log exceeds max limit. 533 Informational BMC 3000 15 534 Predictive BMC 3000 30 535 Unrecoverable BMC 3000 30 536 Informational BMC 1500, Predictive BMC 1500 45 537 Informational BMC 1500, Unrecoverable BMC 1500 45 538 Unrecoverable BMC 1500, Predictive BMC 1500 30 539 540 541Verify Error Logging Rotation Policy With All Types Of Errors 542 [Documentation] Verify error logging rotation policy with all types of errors. 543 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Errors 544 [Template] Error Logging Rotation Policy 545 546 # Error logs to be created % of total logging space when error 547 # log exceeds max limit. 548 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 549 550 551Verify Error Logging Rotation Policy With HOST Error Logs 552 [Documentation] Verify error logging rotation policy for non bmc error logs. 553 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 554 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 555 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 556 [Template] Error Logging Rotation Policy 557 558 # Error logs to be created % of total logging space when error 559 # log exceeds max limit. 560 Unrecoverable HOST 3000 30 561 Unrecoverable HOST 1500, Informational BMC 1500 45 562 Unrecoverable HOST 1500, Predictive BMC 1500 60 563 564 565Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 566 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 567 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 568 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 569 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 570 [Template] Error Logging Rotation Policy 571 572 # Error logs to be created % of total logging space when error 573 # log exceeds max limit. 574 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 575 576 577Verify Reverse Order Of PEL Logs 578 [Documentation] Verify PEL command to output PEL logs in reverse order. 579 [Tags] Verify_Reverse_PEL_Logs 580 581 Redfish Purge Event Log 582 583 # Below commands create unrecoverable error log at first and then the predictable error. 584 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 585 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 586 587 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 588 ${pel_records}= peltool -lr 589 590 # It is found that, variables like dictionary always keep items in sorted order that makes 591 # this verification not possible, hence json is used to keep the items original order. 592 ${pel_records}= Convert To String ${pel_records} 593 ${json_string}= Replace String ${pel_records} ' " 594 ${json_object}= Evaluate json.loads('''${json_string}''') json 595 596 ${list}= Convert To List ${json_object} 597 598 ${id1}= Get From List ${list} 0 599 ${id1}= Convert To Integer ${id1} 600 ${id2}= Get From List ${list} 1 601 ${id2}= Convert To Integer ${id2} 602 603 Should Be True ${id1} > ${id2} 604 605 606Verify Total PEL Count 607 [Documentation] Verify total PEL count returned by peltool command. 608 [Tags] Verify_Total_PEL_Count 609 610 # Initially remove all logs. 611 Redfish Purge Event Log 612 613 # Generate a random number between 1-20. 614 ${random}= Evaluate random.randint(1, 20) modules=random 615 616 # Generate predictive error log multiple times. 617 FOR ${count} IN RANGE 0 ${random} 618 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 619 END 620 621 # Check PEL log count via peltool command and compare it with actual generated log count. 622 ${pel_records}= peltool -n 623 624 Should Be Equal ${pel_records['Number of PELs found']} ${random} 625 626 627Verify Listing Information Error 628 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 629 [Tags] Verify_Listing_Information_Error 630 631 # Initially remove all logs. 632 Redfish Purge Event Log 633 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 634 635 # Generate informational logs and verify that it would not get listed by peltool's list command. 636 ${pel_records}= peltool -l 637 ${ids}= Get Dictionary Keys ${pel_records} 638 Should Be Empty ${ids} 639 640 # Verify that information logs get listed using peltool's list command with -lfh option. 641 ${pel_records}= peltool -lfh 642 ${ids}= Get Dictionary Keys ${pel_records} 643 Should Not Be Empty ${ids} 644 ${id}= Get From List ${ids} 0 645 Should Contain ${pel_records['${id}']['Sev']} Informational 646 647 648Verify PEL Delete 649 [Documentation] Verify that peltool command can delete PEL log based on id. 650 [Tags] Verify_PEL_Delete 651 652 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 653 ${pel_ids}= Get PEL Log Via BMC CLI 654 ${id}= Get From List ${pel_ids} -1 655 Peltool -d ${id} False 656 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 657 658 659*** Keywords *** 660 661Error Logging Rotation Policy 662 [Documentation] Verify that when maximum log limit is reached, given error logging type 663 ... are deleted when reached their max allocated space. 664 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 665 666 # Description of argument(s): 667 # error_log Error logs to be created (E.g. Informational BMC 3000 668 # stands for BMC created 3000 informational error logs). 669 # max_allocated_space_percentage The maximum percentage of disk usage for given error 670 # log type when maximum count/log size is reached. 671 # The maximum error log count is 3000. 672 673 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 674 # in BMC during the test. 675 Redfish Purge Event Log 676 677 @{lists}= Split String ${error_log_type} ,${SPACE} 678 679 ${length}= Get Length ${lists} 680 681 FOR ${list} IN RANGE ${length} 682 @{words}= Split String ${lists}[${list}] ${SPACE} 683 Create Error Log ${words}[0] ${words}[1] ${words}[2] 684 END 685 686 # Create an additional error log to exceed max error logs limit. 687 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 688 689 # Delay for BMC to perform delete older error logs when log limit exceeds. 690 Sleep 10s 691 692 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 693 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 694 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 695 696 ${disk_usage_percentage}= Get Disk Usage For Error Logs 697 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 698 ${percent_diff}= Evaluate abs(${percent_diff}) 699 Should Be True ${percent_diff} <= 0.5 700 701 702Create Error Log 703 [Arguments] ${error_severity} ${error_creator} ${count} 704 705 # Description of argument(s): 706 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 707 # error_creator Name of error log's creator(E.g BMC, Host Boot) 708 # count Number of error logs to be generated. 709 710 FOR ${i} IN RANGE 0 ${count} 711 ${cmd}= Set Variable If 712 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 713 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 714 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 715 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 716 BMC Execute Command ${cmd} 717 END 718 719 720Get Disk Usage For Error Logs 721 [Documentation] Get disk usage percentage for error logs. 722 723 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 724 725 ${usage_output}= Fetch From Left ${usage_output} \/ 726 727 # Covert disk usage unit from KB to MB. 728 ${usage_output}= Evaluate ${usage_output} / 1024 729 730 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 731 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 732 733 [return] ${usage_percent} 734 735 736Create Test PEL Log 737 [Documentation] Generate test PEL log. 738 [Arguments] ${pel_type}=Internal Failure 739 740 # Description of argument(s): 741 # pel_type The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout). 742 743 # Test PEL log entry example: 744 # { 745 # "0x5000002D": { 746 # "SRC": "BD8D1002", 747 # "Message": "An application had an internal failure", 748 # "PLID": "0x5000002D", 749 # "CreatorID": "BMC", 750 # "Subsystem": "BMC Firmware", 751 # "Commit Time": "02/25/2020 04:47:09", 752 # "Sev": "Unrecoverable Error", 753 # "CompID": "0x1000" 754 # } 755 # } 756 757 Run Keyword If '${pel_type}' == 'Internal Failure' 758 ... BMC Execute Command ${CMD_INTERNAL_FAILURE} 759 ... ELSE IF '${pel_type}' == 'FRU Callout' 760 ... BMC Execute Command ${CMD_FRU_CALLOUT} 761 ... ELSE IF '${pel_type}' == 'Procedure And Symbolic FRU Callout' 762 ... BMC Execute Command ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} 763 764 765Get PEL Log IDs 766 [Documentation] Returns the list of PEL log IDs which contains given field's value. 767 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 768 769 # Description of argument(s): 770 # pel_section The section of PEL (e.g. Private Header, User Header). 771 # pel_field The PEL field (e.g. Event Severity, Event Type). 772 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 773 774 ${pel_ids}= Get PEL Log Via BMC CLI 775 @{pel_id_list}= Create List 776 777 FOR ${id} IN @{pel_ids} 778 ${pel_output}= Peltool -i ${id} 779 # Example of PEL output from "peltool -i <id>" command. 780 # [Private Header]: 781 # [Created at]: 08/24/1928 12:04:06 782 # [Created by]: 0x584D 783 # [Sub-section type]: 0 784 # [Entry Id]: 0x50000BB7 785 # [Platform Log Id]: 0x8200061D 786 # [CSSVER]: 787 # [Section Version]: 1 788 # [Creator Subsystem]: PHYP 789 # [BMC Event Log Id]: 341 790 # [Committed at]: 03/25/1920 12:06:22 791 # [User Header]: 792 # [Log Committed by]: 0x4552 793 # [Action Flags]: 794 # [0]: Report Externally 795 # [Subsystem]: I/O Subsystem 796 # [Event Type]: Miscellaneous, Informational Only 797 # [Sub-section type]: 0 798 # [Event Scope]: Entire Platform 799 # [Event Severity]: Informational Event 800 # [Host Transmission]: Not Sent 801 # [Section Version]: 1 802 803 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 804 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 805 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 806 END 807 Sort List ${pel_id_list} 808 809 [Return] ${pel_id_list} 810 811 812Get PEL Log Via BMC CLI 813 [Documentation] Returns the list of PEL IDs using BMC CLI. 814 815 ${pel_records}= Peltool -l 816 ${ids}= Get Dictionary Keys ${pel_records} 817 Sort List ${ids} 818 819 [Return] ${ids} 820 821 822Get PEL Field Value 823 [Documentation] Returns the value of given PEL's field. 824 [Arguments] ${pel_id} ${pel_section} ${pel_field} 825 826 # Description of argument(s): 827 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 828 # pel_section The section of PEL (e.g. Private Header, User Header) 829 # pel_field The PEL field (e.g. Event Severity, Event Type). 830 831 ${pel_output}= Peltool -i ${pel_id} 832 833 # Example of PEL output from "peltool -i <id>" command. 834 # [Private Header]: 835 # [Created at]: 08/24/1928 12:04:06 836 # [Created by]: 0x584D 837 # [Sub-section type]: 0 838 # [Entry Id]: 0x50000BB7 839 # [Platform Log Id]: 0x8200061D 840 # [CSSVER]: 841 # [Section Version]: 1 842 # [Creator Subsystem]: PHYP 843 # [BMC Event Log Id]: 341 844 # [Committed at]: 03/25/1920 12:06:22 845 # [User Header]: 846 # [Log Committed by]: 0x4552 847 # [Action Flags]: 848 # [0]: Report Externally 849 # [Subsystem]: I/O Subsystem 850 # [Event Type]: Miscellaneous, Informational Only 851 # [Sub-section type]: 0 852 # [Event Scope]: Entire Platform 853 # [Event Severity]: Informational Event 854 # [Host Transmission]: Not Sent 855 # [Section Version]: 1 856 857 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 858 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 859 860 [Return] ${pel_field_output} 861