1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14Force Tags Bmc_Pel 15 16*** Variables *** 17 18@{mandatory_pel_fields} Private Header User Header Primary SRC 19... Extended User Header Failing MTMS 20@{mandatory_Predictive_pel_fields} Private Header User Header Primary SRC 21... Extended User Header Failing MTMS User Data 22@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 23... Extended User Header User Data 24 25*** Test Cases *** 26 27Create Test PEL Log And Verify 28 [Documentation] Create PEL log using busctl command and verify via peltool. 29 [Tags] Create_Test_PEL_Log_And_Verify 30 31 Redfish Purge Event Log 32 Create Test PEL Log 33 ${pel_id}= Get PEL Log Via BMC CLI 34 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 35 36 37Verify PEL Log Details 38 [Documentation] Verify PEL log details via peltool. 39 [Tags] Verify_PEL_Log_Details 40 41 Redfish Purge Event Log 42 43 ${bmc_time1}= CLI Get BMC DateTime 44 Create Test PEL Log 45 ${bmc_time2}= CLI Get BMC DateTime 46 47 ${pel_records}= Peltool -l 48 49 # Example output from 'Peltool -l': 50 # pel_records: 51 # [0x50000012]: 52 # [CreatorID]: BMC 53 # [CompID]: 0x1000 54 # [PLID]: 0x50000012 55 # [Subsystem]: BMC Firmware 56 # [Message]: An application had an internal failure 57 # [SRC]: BD8D1002 58 # [Commit Time]: 03/02/2020 09:35:15 59 # [Sev]: Unrecoverable Error 60 61 ${ids}= Get Dictionary Keys ${pel_records} 62 ${id}= Get From List ${ids} 0 63 64 @{pel_fields}= Create List CreatorID Subsystem Message Sev 65 FOR ${field} IN @{pel_fields} 66 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 67 END 68 69 Valid Value pel_records['${id}']['PLID'] ['${id}'] 70 71 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 72 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 73 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 74 75 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 76 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 77 78 # Convert BMC and PEL time to epoch time before comparing. 79 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 80 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 81 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 82 83 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 84 85 86Verify Mandatory Sections Of Error Log PEL 87 [Documentation] Verify mandatory sections of error log PEL. 88 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 89 90 Create Test PEL Log 91 92 ${pel_ids}= Get PEL Log Via BMC CLI 93 ${pel_id}= Get From List ${pel_ids} -1 94 ${pel_output}= Peltool -i ${pel_id} 95 ${pel_sections}= Get Dictionary Keys ${pel_output} 96 97 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fields} 98 99 100Verify PEL Log Persistence After BMC Reboot 101 [Documentation] Verify PEL log persistence after BMC reboot. 102 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 103 104 Create Test PEL Log 105 ${pel_before_reboot}= Get PEL Log Via BMC CLI 106 107 Redfish OBMC Reboot (off) 108 ${pel_after_reboot}= Get PEL Log Via BMC CLI 109 110 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 111 112 113Verify PEL ID Numbering 114 [Documentation] Verify PEL ID numbering. 115 [Tags] Verify_PEL_ID_Numbering 116 117 Redfish Purge Event Log 118 Create Test PEL Log 119 Create Test PEL Log 120 121 ${pel_ids}= Get PEL Log Via BMC CLI 122 123 # Example of PEL IDs from PEL logs. 124 # [0x50000012]: <--- First PEL ID 125 # [CreatorID]: BMC 126 # [CompID]: 0x1000 127 # [PLID]: 0x50000012 128 # [Subsystem]: BMC Firmware 129 # [Message]: An application had an internal failure 130 # [SRC]: BD8D1002 131 # [Commit Time]: 03/02/2020 09:35:15 132 # [Sev]: Unrecoverable Error 133 # 134 # [0x50000013]: <--- Second PEL ID 135 # [CreatorID]: BMC 136 # [CompID]: 0x1000 137 # [PLID]: 0x50000013 138 # [Subsystem]: BMC Firmware 139 # [Message]: An application had an internal failure 140 # [SRC]: BD8D1002 141 # [Commit Time]: 03/02/2020 09:35:15 142 # [Sev]: Unrecoverable Error 143 144 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 145 146Verify Machine Type Model And Serial Number 147 [Documentation] Verify machine type model and serial number from PEL. 148 [Tags] Verify_Machine_Type_Model_And_Serial_Number 149 150 Create Test PEL Log 151 152 ${pel_ids}= Get PEL Log Via BMC CLI 153 ${id}= Get From List ${pel_ids} -1 154 155 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 156 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 157 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 158 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 159 160 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 161 # [Failing MTMS]: 162 # [Created by]: 0x2000 163 # [Machine Type Model]: 1234-ABC <---- Machine type 164 # [Section Version]: 1 165 # [Serial Number]: ABCDEFG <---- Serial number 166 # [Sub-section type]: 0 167 168 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 169 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 170 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 171 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 172 173 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 174 Valid Value pel_serial_number ['${redfish_serial_number}'] 175 176 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 177 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 178 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 179 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 180 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 181 182 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 183 Valid Value pel_serial_number ['${redfish_serial_number}'] 184 185 186Verify Host Off State From PEL 187 [Documentation] Verify Host off state from PEL. 188 [Tags] Verify_Host_Off_State_From_PEL 189 190 Redfish Power Off stack_mode=skip 191 Create Test PEL Log 192 193 ${pel_ids}= Get PEL Log Via BMC CLI 194 ${id}= Get From List ${pel_ids} -1 195 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 196 197 Valid Value pel_host_state ['Off'] 198 199 200Verify Host On State From PEL 201 [Documentation] Verify Host on state from PEL. 202 [Tags] Verify_Host_On_State_From_PEL 203 204 Redfish Power On stack_mode=skip 205 Redfish Purge Event Log 206 207 Create Test PEL Log 208 209 ${pel_ids}= Get PEL Log Via BMC CLI 210 ${id}= Get From List ${pel_ids} -1 211 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 212 213 Valid Value pel_host_state ['Running'] 214 215 216Verify BMC Version From PEL 217 [Documentation] Verify BMC Version from PEL. 218 [Tags] Verify_BMC_Version_From_PEL 219 220 Create Test PEL Log 221 222 ${pel_ids}= Get PEL Log Via BMC CLI 223 ${id}= Get From List ${pel_ids} -1 224 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 225 226 ${bmc_version}= Get BMC Version 227 Valid Value bmc_version ['${bmc_version}'] 228 229 230Verify PEL Log After Host Poweron 231 [Documentation] Verify PEL log generation while booting host. 232 [Tags] Verify_PEL_Log_After_Host_Poweron 233 234 Redfish Power Off stack_mode=skip 235 Redfish Purge Event Log 236 Redfish Power On stack_mode=skip 237 238 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 239 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 240 241 # Get BMC created non-infomational error. 242 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 243 244 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 245 246 247Verify BMC Event Log ID 248 [Documentation] Verify BMC Event Log ID from PEL. 249 [Tags] Verify_BMC_Event_Log_ID 250 251 Redfish Purge Event Log 252 Create Test PEL Log 253 254 ${pel_ids}= Get PEL Log Via BMC CLI 255 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 256 257 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 258 # [Private Header]: 259 # [Created at]: 08/24/1928 12:04:06 260 # [Created by]: 0x584D 261 # [Sub-section type]: 0 262 # [Entry Id]: 0x50000BB7 263 # [Platform Log Id]: 0x8200061D 264 # [CSSVER]: 265 # [Section Version]: 1 266 # [Creator Subsystem]: PHYP 267 # [BMC Event Log Id]: 341 <---- BMC event log id value 268 # [Committed at]: 03/25/1920 12:06:22 269 270 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 271 272 # Example of redfish_event_logs output: 273 # redfish_event_logs: 274 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 275 # [Name]: System Event Log Entries 276 # [Members@odata.count]: 1 277 # [@odata.type]: #LogEntryCollection.LogEntryCollection 278 # [Description]: Collection of System Event Log Entries 279 # [Members]: 280 # [0]: 281 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 282 # [Name]: System Event Log Entry 283 # [Severity]: Critical 284 # [EntryType]: Event 285 # [Created]: 2020-04-02T07:25:13+00:00 286 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 287 # [Id]: 235 <----- Event log ID 288 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 289 290 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 291 292 293Verify FRU Callout 294 [Documentation] Verify FRU callout entries from PEL log. 295 [Tags] Verify_FRU_Callout 296 297 # Power off the system to avoid unnecessary error logs during test. 298 Redfish Power Off stack_mode=skip 299 Create Test PEL Log FRU Callout 300 301 ${pel_ids}= Get PEL Log Via BMC CLI 302 ${id}= Get From List ${pel_ids} -1 303 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 304 305 # Example of PEL Callout Section from "peltool -i <id>" command. 306 # [Callouts]: 307 # [0]: 308 # [FRU Type]: Normal Hardware FRU 309 # [Priority]: Mandatory, replace all with this type as a unit 310 # [Location Code]: U78DA.ND1.1234567-P0 311 # [Part Number]: F191014 312 # [CCIN]: 2E2D 313 # [Serial Number]: YL2E2D010000 314 # [Callout Count]: 1 315 316 Valid Value pel_callout_section['Callout Count'] ['1'] 317 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 318 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 319 320 # Verify Location Code field of PEL callout with motherboard's Location Code. 321 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 322 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 323 ... ${busctl_output[0].split('"')[1].strip('"')} 324 325 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 326 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 327 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 328 329 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 330 ${busctl_output}= BMC Execute Command 331 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 332 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 333 ... ${busctl_output[0].split('"')[1].strip('"')} 334 335 336Verify Procedure And Symbolic FRU Callout 337 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 338 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 339 340 Create Test PEL Log Procedure And Symbolic FRU Callout 341 342 ${pel_ids}= Get PEL Log Via BMC CLI 343 ${id}= Get From List ${pel_ids} -1 344 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 345 346 # Example of PEL Callout Section from "peltool -i <id>" command. 347 # [Callouts]: 348 # [0]: 349 # [Priority]: Mandatory, replace all with this type as a unit 350 # [Procedure Number]: BMCSP02 351 # [FRU Type]: Maintenance Procedure Required 352 # [1]: 353 # [Priority]: Medium Priority 354 # [Part Number]: SVCDOCS 355 # [FRU Type]: Symbolic FRU 356 # [Callout Count]: 2 357 358 Valid Value pel_callout_section['Callout Count'] ['2'] 359 360 # Verify procedural callout info. 361 362 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 363 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 364 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 365 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 366 367 # Verify procedural callout info. 368 369 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 370 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 371 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 372 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 373 374 375Verify PEL Log Entry For Event Log 376 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 377 [Tags] Verify_PEL_Log_Entry_For_Event_Log 378 379 Redfish Purge Event Log 380 381 # Create an unrecoverable error log. 382 Create Test PEL Log Unrecoverable Error 383 384 ${elog_entry}= Get Event Logs 385 # Example of Redfish event logs: 386 # elog_entry: 387 # [0]: 388 # [Message]: BD802003 event in subsystem: Platform Firmware 389 # [Created]: 2023-05-08T10:19:30.412+00:00 390 # [Id]: 1 391 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 392 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 393 # [EntryType]: Event 394 # [Severity]: Critical 395 # [Name]: System Event Log Entry 396 397 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"].split('.')[0]} epoch 398 399 ${pel_records}= Peltool -l 400 # Example output from 'Peltool -l': 401 # pel_records: 402 # [0x50000023]: 403 # [SRC]: BD802003 404 # [CreatorID]: BMC 405 # [Message]: This is a test error 406 # [CompID]: 0x2000 407 # [PLID]: 0x500053D9 408 # [Commit Time]: 04/20/2020 01:55:22 409 # [Subsystem]: Platform Firmware 410 # [Sev]: Unrecoverable Error 411 412 ${ids}= Get Dictionary Keys ${pel_records} 413 ${id}= Get From List ${ids} 0 414 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 415 ... date_format=%m/%d/%Y %H:%M:%S 416 417 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 418 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 419 Should Contain ${elog_entry[0]['Message']} 420 ... ${pel_records['${id}']['SRC']} ignore_case=True 421 422 Should Be Equal ${redfish_log_time} ${pel_log_time} 423 424 425Verify Delete All PEL 426 [Documentation] Verify deleting all PEL logs. 427 [Tags] Verify_Delete_All_PEL 428 429 Create Test PEL Log 430 Create Test PEL Log 431 Peltool --delete-all False 432 433 ${pel_ids}= Get PEL Log Via BMC CLI 434 Should Be Empty ${pel_ids} 435 436 437Verify Informational Error Log 438 [Documentation] Create an informational error log and verify. 439 [Tags] Verify_Informational_Error_Log 440 441 Redfish Purge Event Log 442 # Create an informational error log. 443 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 444 ${pel_records}= Peltool -lfh 445 446 # An example of information error log data: 447 # { 448 # "0x500006A0": { 449 # "SRC": "BD8D1002", 450 # "Message": "An application had an internal failure", 451 # "PLID": "0x500006A0", 452 # "CreatorID": "BMC", 453 # "Subsystem": "BMC Firmware", 454 # "Commit Time": "10/14/2020 11:41:38", 455 # "Sev": "Informational Event", 456 # "CompID": "0x1000" 457 # } 458 # } 459 460 ${ids}= Get Dictionary Keys ${pel_records} 461 ${id}= Get From List ${ids} 0 462 Should Contain ${pel_records['${id}']['Sev']} Informational 463 464 465Verify Predictable Error Log 466 [Documentation] Create a predictive error and verify. 467 [Tags] Verify_Predictable_Error_Log 468 469 # Create a predictable error log. 470 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 471 ${pel_records}= Peltool -l 472 473 # An example of predictive error log data: 474 # { 475 # "0x5000069E": { 476 # "SRC": "BD8D1002", 477 # "Message": "An application had an internal failure", 478 # "PLID": "0x5000069E", 479 # "CreatorID": "BMC", 480 # "Subsystem": "BMC Firmware", 481 # "Commit Time": "10/14/2020 11:40:07", 482 # "Sev": "Predictive Error", 483 # "CompID": "0x1000" 484 # } 485 # } 486 487 ${pel_ids}= Get PEL Log Via BMC CLI 488 ${id}= Get From List ${pel_ids} -1 489 Should Contain ${pel_records['${id}']['Sev']} Predictive 490 491 492Verify Unrecoverable Error Log 493 [Documentation] Create an unrecoverable error and verify. 494 [Tags] Verify_Unrecoverable_Error_Log 495 496 # Create an internal failure error log. 497 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 498 ${pel_records}= Peltool -l 499 500 # An example of unrecoverable error log data: 501 # { 502 # "0x50000CC5": { 503 # "SRC": "BD8D1002", 504 # "Message": "An application had an internal failure", 505 # "PLID": "0x50000CC5", 506 # "CreatorID": "BMC", 507 # "Subsystem": "BMC Firmware", 508 # "Commit Time": "04/01/2020 16:44:55", 509 # "Sev": "Unrecoverable Error", 510 # "CompID": "0x1000" 511 # } 512 # } 513 514 ${pel_ids}= Get PEL Log Via BMC CLI 515 ${id}= Get From List ${pel_ids} -1 516 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 517 518 519Verify Error Logging Rotation Policy 520 [Documentation] Verify error logging rotation policy. 521 [Tags] Verify_Error_Logging_Rotation_Policy 522 [Template] Error Logging Rotation Policy 523 524 # Error logs to be created % of total logging space when error 525 # log exceeds max limit. 526 Informational BMC 3000 15 527 Predictive BMC 3000 30 528 Unrecoverable BMC 3000 30 529 Informational BMC 1500, Predictive BMC 1500 45 530 Informational BMC 1500, Unrecoverable BMC 1500 45 531 Unrecoverable BMC 1500, Predictive BMC 1500 30 532 533 534Verify Error Logging Rotation Policy With All Types Of Errors 535 [Documentation] Verify error logging rotation policy with all types of errors. 536 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 537 [Template] Error Logging Rotation Policy 538 539 # Error logs to be created % of total logging space when error 540 # log exceeds max limit. 541 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 542 543 544Verify Error Logging Rotation Policy With HOST Error Logs 545 [Documentation] Verify error logging rotation policy for non bmc error logs. 546 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 547 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 548 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 549 ... /tmp/FILE_HOST_INFORMATIONAL 550 [Template] Error Logging Rotation Policy 551 552 # Error logs to be created % of total logging space when error 553 # log exceeds max limit. 554 Informational HOST 3000 15 555 Unrecoverable HOST 3000 30 556 Informational HOST 1500, Informational BMC 1500 30 557 Informational HOST 1500, Unrecoverable BMC 1500 45 558 Unrecoverable HOST 1500, Informational BMC 1500 45 559 Unrecoverable HOST 1500, Predictive BMC 1500 60 560 561 562Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 563 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 564 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 565 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 566 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 567 [Template] Error Logging Rotation Policy 568 569 # Error logs to be created % of total logging space when error 570 # log exceeds max limit. 571 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 572 573 574Verify Old Logs Are Deleted When Count Crosses Max 575 [Documentation] Verify that when the count crosses max, older logs are deleted. 576 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 577 578 Redfish Purge Event Log 579 # Create 3000 error logs. 580 FOR ${count} IN RANGE ${3000} 581 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 582 END 583 584 # Retrieve the IDs of the logs. 585 ${pel_ids}= Get PEL Log Via BMC CLI 586 ${1st_id}= Get From List ${pel_ids} 0 587 ${3000th_id}= Get From List ${pel_ids} 2999 588 589 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 590 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 591 592 # Wait few seconds for error logs rotation to complete. 593 Sleep 10s 594 595 # Now verify that first log is no more available but the 3000th is available. 596 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 597 Should Be True '${status}' == 'FAIL' 598 Should Contain ${output} PEL not found 599 600 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 601 Should Be True '${status}' == 'PASS' 602 Should Not Contain ${output} PEL not found 603 604 605Verify Reverse Order Of PEL Logs 606 [Documentation] Verify PEL command to output PEL logs in reverse order. 607 [Tags] Verify_Reverse_Order_Of_PEL_Logs 608 609 Redfish Purge Event Log 610 611 # Below commands create unrecoverable error log at first and then the predictable error. 612 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 613 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 614 615 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 616 ${pel_records}= peltool -lr 617 618 # It is found that, variables like dictionary always keep items in sorted order that makes 619 # this verification not possible, hence json is used to keep the items original order. 620 ${pel_records}= Convert To String ${pel_records} 621 ${json_string}= Replace String ${pel_records} ' " 622 ${json_object}= Evaluate json.loads('''${json_string}''') json 623 624 ${list}= Convert To List ${json_object} 625 626 ${id1}= Get From List ${list} 0 627 ${id1}= Convert To Integer ${id1} 628 ${id2}= Get From List ${list} 1 629 ${id2}= Convert To Integer ${id2} 630 631 Should Be True ${id1} > ${id2} 632 633 634Verify Total PEL Count 635 [Documentation] Verify total PEL count returned by peltool command. 636 [Tags] Verify_Total_PEL_Count 637 638 # Initially remove all logs. 639 Redfish Purge Event Log 640 641 # Generate a random number between 1-20. 642 ${random}= Evaluate random.randint(1, 20) modules=random 643 644 # Generate predictive error log multiple times. 645 FOR ${count} IN RANGE 0 ${random} 646 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 647 END 648 649 # Check PEL log count via peltool command and compare it with actual generated log count. 650 ${pel_records}= peltool -n 651 652 Should Be Equal ${pel_records['Number of PELs found']} ${random} 653 654 655Verify Listing Information Error 656 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 657 [Tags] Verify_Listing_Information_Error 658 659 # Initially remove all logs. 660 Redfish Purge Event Log 661 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 662 663 # Generate informational logs and verify that it would not get listed by peltool's list command. 664 ${pel_records}= peltool -l 665 ${ids}= Get Dictionary Keys ${pel_records} 666 Should Be Empty ${ids} 667 668 # Verify that information logs get listed using peltool's list command with -lfh option. 669 ${pel_records}= peltool -lfh 670 ${ids}= Get Dictionary Keys ${pel_records} 671 Should Not Be Empty ${ids} 672 ${id}= Get From List ${ids} 0 673 Should Contain ${pel_records['${id}']['Sev']} Informational 674 675 676Verify PEL Delete 677 [Documentation] Verify that peltool command can delete PEL log based on id. 678 [Tags] Verify_PEL_Delete 679 680 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 681 ${pel_ids}= Get PEL Log Via BMC CLI 682 ${id}= Get From List ${pel_ids} -1 683 Peltool -d ${id} False 684 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 685 686 687Verify Mandatory Fields For Predictive Error 688 [Documentation] Verify mandatory fields of predictive errors from pel information. 689 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 690 691 # Inject predictive error. 692 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 693 694 ${pel_ids}= Get PEL Log Via BMC CLI 695 ${pel_id}= Get From List ${pel_ids} -1 696 ${pel_output}= Peltool -i ${pel_id} 697 # Get all fields in predictive error log. 698 ${pel_sections}= Get Dictionary Keys ${pel_output} 699 700 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fields} 701 702 703Verify Mandatory Fields For Informational Error 704 [Documentation] Verify mandatory fields of informational error from pel information. 705 [Tags] Verify_Mandatory_Fields_For_Informational_Error 706 707 # Inject informational error. 708 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 709 ${pel_records}= Peltool -lfh 710 711 ${ids}= Get Dictionary Keys ${pel_records} 712 ${pel_id}= Get From List ${ids} -1 713 ${pel_output}= Peltool -i ${pel_id} 714 715 # Get all fields in the informational error log. 716 ${pel_sections}= Get Dictionary Keys ${pel_output} 717 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 718 ${contains}= Evaluate "${section}" in "${pel_sections}" 719 Should Be True ${contains} 720 END 721 722 723Verify PEL Log Offloaded To Host 724 [Documentation] Verify host transmission state of offloaded PEL logs. 725 [Tags] Verify_PEL_Log_Offloaded_To_Host 726 [Template] Verify PEL Transmission To Host 727 728 # error_type host_state expected_transmission_state 729 predictive_error On Acked 730 unrecoverable_error On Acked 731 informational_error On Acked 732 733 734Verify PEL Log Not Offloaded To Host 735 [Documentation] Verify host transmission state of not offloaded PEL logs. 736 [Tags] Verify_PEL_Log_Not_Offloaded_To_Host 737 [Template] Verify PEL Transmission To Host 738 739 # error_type host_state expected_transmission_state 740 predictive_error Off Not Sent 741 unrecoverable_error Off Not Sent 742 informational_error Off Not Sent 743 744 745*** Keywords *** 746 747Error Logging Rotation Policy 748 [Documentation] Verify that when maximum log limit is reached, given error logging type 749 ... are deleted when reached their max allocated space. 750 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 751 752 # Description of argument(s): 753 # error_log Error logs to be created (E.g. Informational BMC 3000 754 # stands for BMC created 3000 informational error logs). 755 # max_allocated_space_percentage The maximum percentage of disk usage for given error 756 # log type when maximum count/log size is reached. 757 # The maximum error log count is 3000. 758 759 Redfish.Login 760 761 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 762 # in BMC during the test. 763 Redfish Purge Event Log 764 765 @{lists}= Split String ${error_log_type} ,${SPACE} 766 767 ${length}= Get Length ${lists} 768 769 FOR ${list} IN RANGE ${length} 770 @{words}= Split String ${lists}[${list}] ${SPACE} 771 Create Error Log ${words}[0] ${words}[1] ${words}[2] 772 END 773 774 # Create an additional error log to exceed max error logs limit. 775 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 776 777 # Delay for BMC to perform delete older error logs when log limit exceeds. 778 Sleep 10s 779 780 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 781 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 782 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 783 784 ${disk_usage_percentage}= Get Disk Usage For Error Logs 785 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 786 ${percent_diff}= Evaluate abs(${percent_diff}) 787 788 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 789 ... Evaluate ${percent_diff} <= 0.5 790 ... ELSE 791 ... Set Variable True 792 793 # Check PEL log count via peltool command and compare it with actual generated log count. 794 ${pel_records}= peltool -n 795 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 796 # Number of logs can be 80% of the total logs created after trimming. 797 ${expected_max_record}= Evaluate 3000 * 0.8 798 799 Run Keyword If ${trimmed_as_expected} == False 800 ... Should Be True ${no_pel_records} <= ${expected_max_record} 801 802 803Create Error Log 804 [Documentation] Create an error log. 805 [Arguments] ${error_severity} ${error_creator} ${count} 806 807 # Description of argument(s): 808 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 809 # error_creator Name of error log's creator(E.g BMC, Host Boot) 810 # count Number of error logs to be generated. 811 812 FOR ${i} IN RANGE 0 ${count} 813 ${cmd}= Set Variable If 814 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 815 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 816 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 817 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 818 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 819 BMC Execute Command ${cmd} 820 END 821 822 823Get Disk Usage For Error Logs 824 [Documentation] Get disk usage percentage for error logs. 825 826 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 827 828 ${usage_output}= Fetch From Left ${usage_output} \/ 829 830 # Convert disk usage unit from KB to MB. 831 ${usage_output}= Evaluate ${usage_output} / 1024 832 833 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 834 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 835 836 [return] ${usage_percent} 837 838 839Get PEL Log IDs 840 [Documentation] Returns the list of PEL log IDs which contains given field's value. 841 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 842 843 # Description of argument(s): 844 # pel_section The section of PEL (e.g. Private Header, User Header). 845 # pel_field The PEL field (e.g. Event Severity, Event Type). 846 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 847 848 ${pel_ids}= Get PEL Log Via BMC CLI 849 @{pel_id_list}= Create List 850 851 FOR ${id} IN @{pel_ids} 852 ${pel_output}= Peltool -i ${id} 853 # Example of PEL output from "peltool -i <id>" command. 854 # [Private Header]: 855 # [Created at]: 08/24/1928 12:04:06 856 # [Created by]: 0x584D 857 # [Sub-section type]: 0 858 # [Entry Id]: 0x50000BB7 859 # [Platform Log Id]: 0x8200061D 860 # [CSSVER]: 861 # [Section Version]: 1 862 # [Creator Subsystem]: PHYP 863 # [BMC Event Log Id]: 341 864 # [Committed at]: 03/25/1920 12:06:22 865 # [User Header]: 866 # [Log Committed by]: 0x4552 867 # [Action Flags]: 868 # [0]: Report Externally 869 # [Subsystem]: I/O Subsystem 870 # [Event Type]: Miscellaneous, Informational Only 871 # [Sub-section type]: 0 872 # [Event Scope]: Entire Platform 873 # [Event Severity]: Informational Event 874 # [Host Transmission]: Not Sent 875 # [Section Version]: 1 876 877 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 878 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 879 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 880 END 881 Sort List ${pel_id_list} 882 883 [Return] ${pel_id_list} 884 885 886Get PEL Log Via BMC CLI 887 [Documentation] Returns the list of PEL IDs using BMC CLI. 888 889 ${pel_records}= Peltool -l 890 ${ids}= Get Dictionary Keys ${pel_records} 891 Sort List ${ids} 892 893 [Return] ${ids} 894 895 896Get PEL Field Value 897 [Documentation] Returns the value of given PEL's field. 898 [Arguments] ${pel_id} ${pel_section} ${pel_field} 899 900 # Description of argument(s): 901 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 902 # pel_section The section of PEL (e.g. Private Header, User Header) 903 # pel_field The PEL field (e.g. Event Severity, Event Type). 904 905 ${pel_output}= Peltool -i ${pel_id} 906 907 # Example of PEL output from "peltool -i <id>" command. 908 # [Private Header]: 909 # [Created at]: 08/24/1928 12:04:06 910 # [Created by]: 0x584D 911 # [Sub-section type]: 0 912 # [Entry Id]: 0x50000BB7 913 # [Platform Log Id]: 0x8200061D 914 # [CSSVER]: 915 # [Section Version]: 1 916 # [Creator Subsystem]: PHYP 917 # [BMC Event Log Id]: 341 918 # [Committed at]: 03/25/1920 12:06:22 919 # [User Header]: 920 # [Log Committed by]: 0x4552 921 # [Action Flags]: 922 # [0]: Report Externally 923 # [Subsystem]: I/O Subsystem 924 # [Event Type]: Miscellaneous, Informational Only 925 # [Sub-section type]: 0 926 # [Event Scope]: Entire Platform 927 # [Event Severity]: Informational Event 928 # [Host Transmission]: Not Sent 929 # [Section Version]: 1 930 931 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 932 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 933 934 [Return] ${pel_field_output} 935 936 937Verify PEL Transmission To Host 938 [Documentation] Inject PEL log of given type and verify its host transmission state. 939 [Arguments] ${error_type} ${host_state} ${expected_transmission_state} 940 941 # Description of argument(s): 942 # error_type Type of error log to be injected. 943 # host_state Host state which is required before error log injection. 944 # expected_transmission_state Expected host transmission state of PEL log. 945 946 # Get system in required state. 947 Run Keyword If '${host_state}' == 'Off' 948 ... Redfish Power Off stack_mode=skip 949 ... ELSE IF '${host_state}' == 'On' 950 ... Run Keywords Redfish Power On stack_mode=skip AND 951 ... Wait For Host To Ping ${OS_HOST} 5 min 10 sec 952 953 Redfish Purge Event Log 954 955 # Inject required error log. 956 Run Keyword If "${error_type}" == "informational_error" 957 ... BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 958 ... ELSE IF "${error_type}" == "unrecoverable_error" 959 ... BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 960 ... ELSE IF "${error_type}" == "predictive_error" 961 ... BMC Execute Command ${CMD_PREDICTIVE_ERROR} 962 963 ${pel_records}= Get Pel Data From Bmc True True 964 ${ids}= Get Dictionary Keys ${pel_records} 965 ${pel_id}= Get From List ${ids} -1 966 967 # Check host transmission state for the cases where PEL is 968 # expected to be offloaded to HOST. 969 Run Keyword If "${expected_transmission_state}" == "Acked" 970 ... Wait Until Keyword Succeeds 15 min 10 sec 971 ... Check If PEL Transmission State Is Expected ${pel_id} Acked 972 973 # Adding delay before checking host transmission for the cases where PEL is 974 # not expected to be offloaded to HOST. 975 Run Keyword If "${expected_transmission_state}" == "Not sent" 976 ... Run Keywords Sleep 120s AND 977 ... Check If PEL Transmission State Is Expected ${pel_id} Not sent 978 979 980Check If PEL Transmission State Is Expected 981 [Documentation] Check if PEL's host transmission state is matching expected state. 982 [Arguments] ${pel_id} ${expected_transmission_state} 983 984 # Description of argument(s): 985 # expected_transmission_state Expected transmission state of PEL log. 986 987 # Check the transmission details in the error log. 988 ${host_transmission}= Get PEL Field Value ${pel_id} User Header Host Transmission 989 Should Be Equal ${host_transmission} ${expected_transmission_state} 990