1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17${CMD_INTERNAL_FAILURE} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 18... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 19... xyz.openbmc_project.Logging.Entry.Level.Error 0 20 21${CMD_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 22... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.Timeout 23... xyz.openbmc_project.Logging.Entry.Level.Error 2 "TIMEOUT_IN_MSEC" "5" 24... "CALLOUT_INVENTORY_PATH" "/xyz/openbmc_project/inventory/system/chassis/motherboard" 25 26${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 27... xyz.openbmc_project.Logging.Create Create ssa{ss} org.open_power.Logging.Error.TestError1 28... xyz.openbmc_project.Logging.Entry.Level.Error 0 29 30${CMD_INFORMATIONAL_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 31... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.TestError2 32... xyz.openbmc_project.Logging.Entry.Level.Informational 0 33 34${CMD_INVENTORY_PREFIX} busctl get-property xyz.openbmc_project.Inventory.Manager 35... /xyz/openbmc_project/inventory/system/chassis/motherboard 36 37${CMD_UNRECOVERABLE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 38... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 39... xyz.openbmc_project.Logging.Entry.Level.Error 0 40 41${CMD_PREDICTIVE_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 42... xyz.openbmc_project.Logging.Create Create ssa{ss} xyz.openbmc_project.Common.Error.InternalFailure 43... xyz.openbmc_project.Logging.Entry.Level.Warning 0 44 45${CMD_UNRECOVERABLE_HOST_ERROR} busctl call xyz.openbmc_project.Logging /xyz/openbmc_project/logging 46... xyz.openbmc_project.Logging.Create Create ssa{ss} 47... xyz.openbmc_project.Host.Error.Event xyz.openbmc_project.Logging.Entry.Level.Error 1 48... RAWPEL /tmp/FILE_NBMC_UNRECOVERABLE 49 50@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 51 52*** Test Cases *** 53 54Create Test PEL Log And Verify 55 [Documentation] Create PEL log using busctl command and verify via peltool. 56 [Tags] Create_Test_PEL_Log_And_Verify 57 58 Redfish Purge Event Log 59 Create Test PEL Log 60 ${pel_id}= Get PEL Log Via BMC CLI 61 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 62 63 64Verify PEL Log Details 65 [Documentation] Verify PEL log details via peltool. 66 [Tags] Verify_PEL_Log_Details 67 68 Redfish Purge Event Log 69 70 ${bmc_time1}= CLI Get BMC DateTime 71 Create Test PEL Log 72 ${bmc_time2}= CLI Get BMC DateTime 73 74 ${pel_records}= Peltool -l 75 76 # Example output from 'Peltool -l': 77 # pel_records: 78 # [0x50000012]: 79 # [CreatorID]: BMC 80 # [CompID]: 0x1000 81 # [PLID]: 0x50000012 82 # [Subsystem]: BMC Firmware 83 # [Message]: An application had an internal failure 84 # [SRC]: BD8D1002 85 # [Commit Time]: 03/02/2020 09:35:15 86 # [Sev]: Unrecoverable Error 87 88 ${ids}= Get Dictionary Keys ${pel_records} 89 ${id}= Get From List ${ids} 0 90 91 @{pel_fields}= Create List CreatorID Subsystem Message Sev 92 FOR ${field} IN @{pel_fields} 93 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 94 END 95 96 Valid Value pel_records['${id}']['PLID'] ['${id}'] 97 98 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 99 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 100 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 101 102 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 103 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 104 105 # Convert BMC and PEL time to epoch time before comparing. 106 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 107 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 108 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 109 110 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 111 112 113Verify Mandatory Sections Of Error Log PEL 114 [Documentation] Verify mandatory sections of error log PEL. 115 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 116 117 Create Test PEL Log 118 119 ${pel_ids}= Get PEL Log Via BMC CLI 120 ${pel_id}= Get From List ${pel_ids} -1 121 ${pel_output}= Peltool -i ${pel_id} 122 ${pel_sections}= Get Dictionary Keys ${pel_output} 123 124 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 125 126 127Verify PEL Log Persistence After BMC Reboot 128 [Documentation] Verify PEL log persistence after BMC reboot. 129 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 130 131 Create Test PEL Log 132 ${pel_before_reboot}= Get PEL Log Via BMC CLI 133 134 Redfish OBMC Reboot (off) 135 ${pel_after_reboot}= Get PEL Log Via BMC CLI 136 137 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 138 139 140Verify PEL ID Numbering 141 [Documentation] Verify PEL ID numbering. 142 [Tags] Verify_PEL_ID_Numbering 143 144 Redfish Purge Event Log 145 Create Test PEL Log 146 Create Test PEL Log 147 148 ${pel_ids}= Get PEL Log Via BMC CLI 149 150 # Example of PEL IDs from PEL logs. 151 # [0x50000012]: <--- First PEL ID 152 # [CreatorID]: BMC 153 # [CompID]: 0x1000 154 # [PLID]: 0x50000012 155 # [Subsystem]: BMC Firmware 156 # [Message]: An application had an internal failure 157 # [SRC]: BD8D1002 158 # [Commit Time]: 03/02/2020 09:35:15 159 # [Sev]: Unrecoverable Error 160 # 161 # [0x50000013]: <--- Second PEL ID 162 # [CreatorID]: BMC 163 # [CompID]: 0x1000 164 # [PLID]: 0x50000013 165 # [Subsystem]: BMC Firmware 166 # [Message]: An application had an internal failure 167 # [SRC]: BD8D1002 168 # [Commit Time]: 03/02/2020 09:35:15 169 # [Sev]: Unrecoverable Error 170 171 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 172 173Verify Machine Type Model And Serial Number 174 [Documentation] Verify machine type model and serial number from PEL. 175 [Tags] Verify_Machine_Type_Model_And_Serial_Number 176 177 Create Test PEL Log 178 179 ${pel_ids}= Get PEL Log Via BMC CLI 180 ${id}= Get From List ${pel_ids} -1 181 182 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 183 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 184 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 185 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 186 187 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 188 # [Failing MTMS]: 189 # [Created by]: 0x2000 190 # [Machine Type Model]: 1234-ABC <---- Machine type 191 # [Section Version]: 1 192 # [Serial Number]: ABCDEFG <---- Serial number 193 # [Sub-section type]: 0 194 195 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 196 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 197 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 198 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 199 200 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 201 Valid Value pel_serial_number ['${redfish_serial_number}'] 202 203 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 204 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 205 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 206 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 207 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 208 209 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 210 Valid Value pel_serial_number ['${redfish_serial_number}'] 211 212 213Verify Host Off State From PEL 214 [Documentation] Verify Host off state from PEL. 215 [Tags] Verify_Host_Off_State_From_PEL 216 217 Redfish Power Off stack_mode=skip 218 Create Test PEL Log 219 220 ${pel_ids}= Get PEL Log Via BMC CLI 221 ${id}= Get From List ${pel_ids} -1 222 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 223 224 Valid Value pel_host_state ['Off'] 225 226 227Verify BMC Version From PEL 228 [Documentation] Verify BMC Version from PEL. 229 [Tags] Verify_BMC_Version_From_PEL 230 231 Create Test PEL Log 232 233 ${pel_ids}= Get PEL Log Via BMC CLI 234 ${id}= Get From List ${pel_ids} -1 235 ${pel_bmc_version}= Get PEL Field Value ${id} User Data BMC Version ID 236 237 ${bmc_version}= Get BMC Version 238 Valid Value bmc_version ['${bmc_version}'] 239 240 241Verify PEL Log After Host Poweron 242 [Documentation] Verify PEL log generation while booting host. 243 [Tags] Verify_PEL_Log_After_Host_Poweron 244 245 Redfish Power Off stack_mode=skip 246 Redfish Purge Event Log 247 Redfish Power On stack_mode=skip 248 249 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 250 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 251 252 # Get BMC created non-infomational error. 253 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 254 255 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 256 257 258Verify BMC Event Log ID 259 [Documentation] Verify BMC Event Log ID from PEL. 260 [Tags] Verify_BMC_Event_Log_ID 261 262 Redfish Purge Event Log 263 Create Test PEL Log 264 265 ${pel_ids}= Get PEL Log Via BMC CLI 266 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 267 268 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 269 # [Private Header]: 270 # [Created at]: 08/24/1928 12:04:06 271 # [Created by]: 0x584D 272 # [Sub-section type]: 0 273 # [Entry Id]: 0x50000BB7 274 # [Platform Log Id]: 0x8200061D 275 # [CSSVER]: 276 # [Section Version]: 1 277 # [Creator Subsystem]: PHYP 278 # [BMC Event Log Id]: 341 <---- BMC event log id value 279 # [Committed at]: 03/25/1920 12:06:22 280 281 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 282 283 # Example of redfish_event_logs output: 284 # redfish_event_logs: 285 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 286 # [Name]: System Event Log Entries 287 # [Members@odata.count]: 1 288 # [@odata.type]: #LogEntryCollection.LogEntryCollection 289 # [Description]: Collection of System Event Log Entries 290 # [Members]: 291 # [0]: 292 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 293 # [Name]: System Event Log Entry 294 # [Severity]: Critical 295 # [EntryType]: Event 296 # [Created]: 2020-04-02T07:25:13+00:00 297 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 298 # [Id]: 235 <----- Event log ID 299 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 300 301 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 302 303 304Verify FRU Callout 305 [Documentation] Verify FRU callout entries from PEL log. 306 [Tags] Verify_FRU_Callout 307 308 Create Test PEL Log FRU Callout 309 310 ${pel_ids}= Get PEL Log Via BMC CLI 311 ${id}= Get From List ${pel_ids} -1 312 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 313 314 # Example of PEL Callout Section from "peltool -i <id>" command. 315 # [Callouts]: 316 # [0]: 317 # [FRU Type]: Normal Hardware FRU 318 # [Priority]: Mandatory, replace all with this type as a unit 319 # [Location Code]: U78DA.ND1.1234567-P0 320 # [Part Number]: F191014 321 # [CCIN]: 2E2D 322 # [Serial Number]: YL2E2D010000 323 # [Callout Count]: 1 324 325 Valid Value pel_callout_section['Callout Count'] ['1'] 326 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 327 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 328 329 # Verify Location Code field of PEL callout with motherboard's Location Code. 330 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 331 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 332 ... ${busctl_output[0].split('"')[1].strip('"')} 333 334 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 335 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 336 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 337 338 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 339 ${busctl_output}= BMC Execute Command 340 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 341 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 342 ... ${busctl_output[0].split('"')[1].strip('"')} 343 344 345Verify Procedure And Symbolic FRU Callout 346 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 347 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 348 349 Create Test PEL Log Procedure And Symbolic FRU Callout 350 351 ${pel_ids}= Get PEL Log Via BMC CLI 352 ${id}= Get From List ${pel_ids} -1 353 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 354 355 # Example of PEL Callout Section from "peltool -i <id>" command. 356 # [Callouts]: 357 # [0]: 358 # [Priority]: Mandatory, replace all with this type as a unit 359 # [Procedure Number]: BMCSP02 360 # [FRU Type]: Maintenance Procedure Required 361 # [1]: 362 # [Priority]: Medium Priority 363 # [Part Number]: SVCDOCS 364 # [FRU Type]: Symbolic FRU 365 # [Callout Count]: 2 366 367 Valid Value pel_callout_section['Callout Count'] ['2'] 368 369 # Verify procedural callout info. 370 371 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 372 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 373 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 374 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 375 376 # Verify procedural callout info. 377 378 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 379 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 380 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 381 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 382 383 384Verify PEL Log Entry For Event Log 385 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 386 [Tags] Verify_PEL_Log_Entry_For_Event_Log 387 388 Redfish Purge Event Log 389 # Create an internal failure error log. 390 BMC Execute Command ${CMD_INTERNAL_FAILURE} 391 392 ${elog_entry}= Get Event Logs 393 # Example of Redfish event logs: 394 # elog_entry: 395 # [0]: 396 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 397 # [Created]: 2020-04-20T01:55:22+00:00 398 # [Id]: 1 399 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 400 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 401 # [EntryType]: Event 402 # [Severity]: Critical 403 # [Name]: System Event Log Entry 404 405 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 406 407 ${pel_records}= Peltool -l 408 # Example output from 'Peltool -l': 409 # pel_records: 410 # [0x50000023]: 411 # [SRC]: BD8D1002 412 # [CreatorID]: BMC 413 # [Message]: An application had an internal failure 414 # [CompID]: 0x1000 415 # [PLID]: 0x50000023 416 # [Commit Time]: 04/20/2020 01:55:22 417 # [Subsystem]: BMC Firmware 418 # [Sev]: Unrecoverable Error 419 420 ${ids}= Get Dictionary Keys ${pel_records} 421 ${id}= Get From List ${ids} 0 422 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 423 ... date_format=%m/%d/%Y %H:%M:%S 424 425 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 426 Should Contain Any ${pel_records['${id}']['Message']} internal failure ignore_case=True 427 Should Contain Any ${elog_entry[0]['Message']} InternalFailure ignore_case=True 428 429 Should Be Equal ${redfish_log_time} ${pel_log_time} 430 431 432Verify Delete All PEL 433 [Documentation] Verify deleting all PEL logs. 434 [Tags] Verify_Delete_All_PEL 435 436 Create Test PEL Log 437 Create Test PEL Log 438 Peltool --delete-all False 439 440 ${pel_ids}= Get PEL Log Via BMC CLI 441 Should Be Empty ${pel_ids} 442 443 444Verify Informational Error Log 445 [Documentation] Create an informational error log and verify. 446 [Tags] Verify_Informational_Error_Log 447 448 Redfish Purge Event Log 449 # Create an informational error log. 450 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 451 ${pel_records}= Peltool -lfh 452 453 # An example of information error log data: 454 # { 455 # "0x500006A0": { 456 # "SRC": "BD8D1002", 457 # "Message": "An application had an internal failure", 458 # "PLID": "0x500006A0", 459 # "CreatorID": "BMC", 460 # "Subsystem": "BMC Firmware", 461 # "Commit Time": "10/14/2020 11:41:38", 462 # "Sev": "Informational Event", 463 # "CompID": "0x1000" 464 # } 465 # } 466 467 ${ids}= Get Dictionary Keys ${pel_records} 468 ${id}= Get From List ${ids} 0 469 Should Contain ${pel_records['${id}']['Sev']} Informational 470 471 472Verify Predictable Error Log 473 [Documentation] Create a predictive error and verify. 474 [Tags] Verify_Predictable_Error_Log 475 476 # Create a predictable error log. 477 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 478 ${pel_records}= Peltool -l 479 480 # An example of predictive error log data: 481 # { 482 # "0x5000069E": { 483 # "SRC": "BD8D1002", 484 # "Message": "An application had an internal failure", 485 # "PLID": "0x5000069E", 486 # "CreatorID": "BMC", 487 # "Subsystem": "BMC Firmware", 488 # "Commit Time": "10/14/2020 11:40:07", 489 # "Sev": "Predictive Error", 490 # "CompID": "0x1000" 491 # } 492 # } 493 494 ${pel_ids}= Get PEL Log Via BMC CLI 495 ${id}= Get From List ${pel_ids} -1 496 Should Contain ${pel_records['${id}']['Sev']} Predictive 497 498 499Verify Unrecoverable Error Log 500 [Documentation] Create an unrecoverable error and verify. 501 [Tags] Verify_Unrecoverable_Error_Log 502 503 # Create an internal failure error log. 504 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 505 ${pel_records}= Peltool -l 506 507 # An example of unrecoverable error log data: 508 # { 509 # "0x50000CC5": { 510 # "SRC": "BD8D1002", 511 # "Message": "An application had an internal failure", 512 # "PLID": "0x50000CC5", 513 # "CreatorID": "BMC", 514 # "Subsystem": "BMC Firmware", 515 # "Commit Time": "04/01/2020 16:44:55", 516 # "Sev": "Unrecoverable Error", 517 # "CompID": "0x1000" 518 # } 519 # } 520 521 ${pel_ids}= Get PEL Log Via BMC CLI 522 ${id}= Get From List ${pel_ids} -1 523 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 524 525 526Verify Error Logging Rotation Policy 527 [Documentation] Verify error logging rotation policy. 528 [Tags] Verify_Error_Logging_Rotation_Policy 529 [Template] Error Logging Rotation Policy 530 531 # Error logs to be created % of total logging space when error 532 # log exceeds max limit. 533 Informational BMC 3000 15 534 Predictive BMC 3000 30 535 Unrecoverable BMC 3000 30 536 Informational BMC 1500, Predictive BMC 1500 45 537 Informational BMC 1500, Unrecoverable BMC 1500 45 538 Unrecoverable BMC 1500, Predictive BMC 1500 30 539 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 540 541 542Verify Error Logging Rotation Policy With HOST Error Logs 543 [Documentation] Verify error logging rotation policy for non bmc error logs. 544 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 545 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 546 ... /tmp/FILE_NBMC_UNRECOVERABLE 547 [Template] Error Logging Rotation Policy 548 549 # Error logs to be created % of total logging space when error 550 # log exceeds max limit. 551 Unrecoverable HOST 3000 30 552 Unrecoverable HOST 1500, Informational BMC 1500 45 553 Unrecoverable HOST 1500, Predictive BMC 1500 60 554 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 555 556 557Verify Reverse Order Of PEL Logs 558 [Documentation] Verify PEL command to output PEL logs in reverse order. 559 [Tags] Verify_Reverse_PEL_Logs 560 561 Redfish Purge Event Log 562 563 # Below commands create unrecoverable error log at first and then the predictable error. 564 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 565 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 566 567 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 568 ${pel_records}= peltool -lr 569 570 # It is found that, variables like dictionary always keep items in sorted order that makes 571 # this verification not possible, hence json is used to keep the items original order. 572 ${pel_records}= Convert To String ${pel_records} 573 ${json_string}= Replace String ${pel_records} ' " 574 ${json_object}= Evaluate json.loads('''${json_string}''') json 575 576 ${list}= Convert To List ${json_object} 577 578 ${id1}= Get From List ${list} 0 579 ${id1}= Convert To Integer ${id1} 580 ${id2}= Get From List ${list} 1 581 ${id2}= Convert To Integer ${id2} 582 583 Should Be True ${id1} > ${id2} 584 585 586Verify Total PEL Count 587 [Documentation] Verify total PEL count returned by peltool command. 588 [Tags] Verify_Total_PEL_Count 589 590 # Initially remove all logs. 591 Redfish Purge Event Log 592 593 # Generate a random number between 1-20. 594 ${random}= Evaluate random.randint(1, 20) modules=random 595 596 # Generate predictive error log multiple times. 597 FOR ${count} IN RANGE 0 ${random} 598 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 599 END 600 601 # Check PEL log count via peltool command and compare it with actual generated log count. 602 ${pel_records}= peltool -n 603 604 Should Be Equal ${pel_records['Number of PELs found']} ${random} 605 606 607Verify Listing Information Error 608 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 609 [Tags] Verify_Listing_Information_Error 610 611 # Initially remove all logs. 612 Redfish Purge Event Log 613 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 614 615 # Generate informational logs and verify that it would not get listed by peltool's list command. 616 ${pel_records}= peltool -l 617 ${ids}= Get Dictionary Keys ${pel_records} 618 Should Be Empty ${ids} 619 620 # Verify that information logs get listed using peltool's list command with -lfh option. 621 ${pel_records}= peltool -lfh 622 ${ids}= Get Dictionary Keys ${pel_records} 623 Should Not Be Empty ${ids} 624 ${id}= Get From List ${ids} 0 625 Should Contain ${pel_records['${id}']['Sev']} Informational 626 627 628Verify PEL Delete 629 [Documentation] Verify that peltool command can delete PEL log based on id. 630 [Tags] Verify_PEL_Delete 631 632 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 633 ${pel_ids}= Get PEL Log Via BMC CLI 634 ${id}= Get From List ${pel_ids} -1 635 Peltool -d ${id} False 636 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 637 638 639*** Keywords *** 640 641Error Logging Rotation Policy 642 [Documentation] Verify that when maximum log limit is reached, given error logging type 643 ... are deleted when reached their max allocated space. 644 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 645 646 # Description of argument(s): 647 # error_log Error logs to be created (E.g. Informational BMC 3000 648 # stands for BMC created 3000 informational error logs). 649 # max_allocated_space_percentage The maximum percentage of disk usage for given error 650 # log type when maximum count/log size is reached. 651 # The maximum error log count is 3000. 652 653 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 654 # in BMC during the test. 655 Redfish Purge Event Log 656 657 @{lists}= Split String ${error_log_type} ,${SPACE} 658 659 ${length}= Get Length ${lists} 660 661 FOR ${list} IN RANGE ${length} 662 @{words}= Split String ${lists}[${list}] ${SPACE} 663 Create Error Log ${words}[0] ${words}[1] ${words}[2] 664 END 665 666 # Create an additional error log to exceed max error logs limit. 667 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 668 669 # Delay for BMC to perform delete older error logs when log limit exceeds. 670 Sleep 10s 671 672 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 673 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 674 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 675 676 ${disk_usage_percentage}= Get Disk Usage For Error Logs 677 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 678 ${percent_diff}= Evaluate abs(${percent_diff}) 679 Should Be True ${percent_diff} <= 0.5 680 681 682Create Error Log 683 [Arguments] ${error_severity} ${error_creator} ${count} 684 685 # Description of argument(s): 686 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 687 # error_creator Name of error log's creator(E.g BMC, Host Boot) 688 # count Number of error logs to be generated. 689 690 FOR ${i} IN RANGE 0 ${count} 691 ${cmd}= Set Variable If 692 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 693 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 694 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 695 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 696 BMC Execute Command ${cmd} 697 END 698 699 700Get Disk Usage For Error Logs 701 [Documentation] Get disk usage percentage for error logs. 702 703 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 704 705 ${usage_output}= Fetch From Left ${usage_output} \/ 706 707 # Covert disk usage unit from KB to MB. 708 ${usage_output}= Evaluate ${usage_output} / 1024 709 710 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 711 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 712 713 [return] ${usage_percent} 714 715 716Create Test PEL Log 717 [Documentation] Generate test PEL log. 718 [Arguments] ${pel_type}=Internal Failure 719 720 # Description of argument(s): 721 # pel_type The PEL type (e.g. Internal Failure, FRU Callout, Procedural Callout). 722 723 # Test PEL log entry example: 724 # { 725 # "0x5000002D": { 726 # "SRC": "BD8D1002", 727 # "Message": "An application had an internal failure", 728 # "PLID": "0x5000002D", 729 # "CreatorID": "BMC", 730 # "Subsystem": "BMC Firmware", 731 # "Commit Time": "02/25/2020 04:47:09", 732 # "Sev": "Unrecoverable Error", 733 # "CompID": "0x1000" 734 # } 735 # } 736 737 Run Keyword If '${pel_type}' == 'Internal Failure' 738 ... BMC Execute Command ${CMD_INTERNAL_FAILURE} 739 ... ELSE IF '${pel_type}' == 'FRU Callout' 740 ... BMC Execute Command ${CMD_FRU_CALLOUT} 741 ... ELSE IF '${pel_type}' == 'Procedure And Symbolic FRU Callout' 742 ... BMC Execute Command ${CMD_PROCEDURAL_SYMBOLIC_FRU_CALLOUT} 743 744 745Get PEL Log IDs 746 [Documentation] Returns the list of PEL log IDs which contains given field's value. 747 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 748 749 # Description of argument(s): 750 # pel_section The section of PEL (e.g. Private Header, User Header). 751 # pel_field The PEL field (e.g. Event Severity, Event Type). 752 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 753 754 ${pel_ids}= Get PEL Log Via BMC CLI 755 @{pel_id_list}= Create List 756 757 FOR ${id} IN @{pel_ids} 758 ${pel_output}= Peltool -i ${id} 759 # Example of PEL output from "peltool -i <id>" command. 760 # [Private Header]: 761 # [Created at]: 08/24/1928 12:04:06 762 # [Created by]: 0x584D 763 # [Sub-section type]: 0 764 # [Entry Id]: 0x50000BB7 765 # [Platform Log Id]: 0x8200061D 766 # [CSSVER]: 767 # [Section Version]: 1 768 # [Creator Subsystem]: PHYP 769 # [BMC Event Log Id]: 341 770 # [Committed at]: 03/25/1920 12:06:22 771 # [User Header]: 772 # [Log Committed by]: 0x4552 773 # [Action Flags]: 774 # [0]: Report Externally 775 # [Subsystem]: I/O Subsystem 776 # [Event Type]: Miscellaneous, Informational Only 777 # [Sub-section type]: 0 778 # [Event Scope]: Entire Platform 779 # [Event Severity]: Informational Event 780 # [Host Transmission]: Not Sent 781 # [Section Version]: 1 782 783 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 784 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 785 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 786 END 787 Sort List ${pel_id_list} 788 789 [Return] ${pel_id_list} 790 791 792Get PEL Log Via BMC CLI 793 [Documentation] Returns the list of PEL IDs using BMC CLI. 794 795 ${pel_records}= Peltool -l 796 ${ids}= Get Dictionary Keys ${pel_records} 797 Sort List ${ids} 798 799 [Return] ${ids} 800 801 802Get PEL Field Value 803 [Documentation] Returns the value of given PEL's field. 804 [Arguments] ${pel_id} ${pel_section} ${pel_field} 805 806 # Description of argument(s): 807 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 808 # pel_section The section of PEL (e.g. Private Header, User Header) 809 # pel_field The PEL field (e.g. Event Severity, Event Type). 810 811 ${pel_output}= Peltool -i ${pel_id} 812 813 # Example of PEL output from "peltool -i <id>" command. 814 # [Private Header]: 815 # [Created at]: 08/24/1928 12:04:06 816 # [Created by]: 0x584D 817 # [Sub-section type]: 0 818 # [Entry Id]: 0x50000BB7 819 # [Platform Log Id]: 0x8200061D 820 # [CSSVER]: 821 # [Section Version]: 1 822 # [Creator Subsystem]: PHYP 823 # [BMC Event Log Id]: 341 824 # [Committed at]: 03/25/1920 12:06:22 825 # [User Header]: 826 # [Log Committed by]: 0x4552 827 # [Action Flags]: 828 # [0]: Report Externally 829 # [Subsystem]: I/O Subsystem 830 # [Event Type]: Miscellaneous, Informational Only 831 # [Sub-section type]: 0 832 # [Event Scope]: Entire Platform 833 # [Event Severity]: Informational Event 834 # [Host Transmission]: Not Sent 835 # [Section Version]: 1 836 837 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 838 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 839 840 [Return] ${pel_field_output} 841