1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14Test Tags Bmc_Pel 15 16*** Variables *** 17 18@{mandatory_pel_fields} Private Header User Header Primary SRC 19... Extended User Header Failing MTMS 20@{mandatory_Predictive_pel_fields} Private Header User Header Primary SRC 21... Extended User Header Failing MTMS User Data 22@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 23... Extended User Header User Data 24 25*** Test Cases *** 26 27Create Test PEL Log And Verify 28 [Documentation] Create PEL log using busctl command and verify via peltool. 29 [Tags] Create_Test_PEL_Log_And_Verify 30 31 Redfish Purge Event Log 32 Create Test PEL Log 33 ${pel_id}= Get PEL Log Via BMC CLI 34 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 35 36 37Verify PEL Log Details 38 [Documentation] Verify PEL log details via peltool. 39 [Tags] Verify_PEL_Log_Details 40 41 Redfish Purge Event Log 42 43 ${bmc_time1}= CLI Get BMC DateTime 44 Create Test PEL Log 45 ${bmc_time2}= CLI Get BMC DateTime 46 47 ${pel_records}= Peltool -l 48 49 # Example output from 'Peltool -l': 50 # pel_records: 51 # [0x50000012]: 52 # [CreatorID]: BMC 53 # [CompID]: 0x1000 54 # [PLID]: 0x50000012 55 # [Subsystem]: BMC Firmware 56 # [Message]: An application had an internal failure 57 # [SRC]: BD8D1002 58 # [Commit Time]: 03/02/2020 09:35:15 59 # [Sev]: Unrecoverable Error 60 61 ${ids}= Get Dictionary Keys ${pel_records} 62 ${id}= Get From List ${ids} 0 63 64 @{pel_fields}= Create List CreatorID Subsystem Message Sev 65 FOR ${field} IN @{pel_fields} 66 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 67 END 68 69 Valid Value pel_records['${id}']['PLID'] ['${id}'] 70 71 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 72 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 73 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 74 75 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 76 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 77 78 # Convert BMC and PEL time to epoch time before comparing. 79 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 80 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 81 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 82 83 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 84 85 86Verify Mandatory Sections Of Error Log PEL 87 [Documentation] Verify mandatory sections of error log PEL. 88 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 89 90 Create Test PEL Log 91 92 ${pel_ids}= Fetch All Pel Ids Based On Error Message test error 93 ${pel_output}= Peltool -i ${pel_ids[0]} 94 ${pel_sections}= Get Dictionary Keys ${pel_output} 95 96 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fields} 97 98 99Verify PEL Log Persistence After BMC Reboot 100 [Documentation] Verify PEL log persistence after BMC reboot. 101 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 102 103 Create Test PEL Log 104 ${pel_before_reboot}= Get PEL Log Via BMC CLI 105 106 Redfish OBMC Reboot (off) 107 ${pel_after_reboot}= Get PEL Log Via BMC CLI 108 109 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 110 111 112Verify PEL ID Numbering 113 [Documentation] Verify PEL ID numbering. 114 [Tags] Verify_PEL_ID_Numbering 115 116 Redfish Purge Event Log 117 Create Test PEL Log 118 Create Test PEL Log 119 120 ${pel_ids}= Get PEL Log Via BMC CLI 121 122 # Example of PEL IDs from PEL logs. 123 # [0x50000012]: <--- First PEL ID 124 # [CreatorID]: BMC 125 # [CompID]: 0x1000 126 # [PLID]: 0x50000012 127 # [Subsystem]: BMC Firmware 128 # [Message]: An application had an internal failure 129 # [SRC]: BD8D1002 130 # [Commit Time]: 03/02/2020 09:35:15 131 # [Sev]: Unrecoverable Error 132 # 133 # [0x50000013]: <--- Second PEL ID 134 # [CreatorID]: BMC 135 # [CompID]: 0x1000 136 # [PLID]: 0x50000013 137 # [Subsystem]: BMC Firmware 138 # [Message]: An application had an internal failure 139 # [SRC]: BD8D1002 140 # [Commit Time]: 03/02/2020 09:35:15 141 # [Sev]: Unrecoverable Error 142 143 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 144 145Verify Machine Type Model And Serial Number 146 [Documentation] Verify machine type model and serial number from PEL. 147 [Tags] Verify_Machine_Type_Model_And_Serial_Number 148 149 Create Test PEL Log 150 151 ${pel_ids}= Get PEL Log Via BMC CLI 152 ${id}= Get From List ${pel_ids} -1 153 154 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 155 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 156 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 157 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 158 159 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 160 # [Failing MTMS]: 161 # [Created by]: 0x2000 162 # [Machine Type Model]: 1234-ABC <---- Machine type 163 # [Section Version]: 1 164 # [Serial Number]: ABCDEFG <---- Serial number 165 # [Sub-section type]: 0 166 167 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/${SYSTEM_ID}/ Model 168 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 169 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/${SYSTEM_ID}/ SerialNumber 170 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 171 172 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 173 Valid Value pel_serial_number ['${redfish_serial_number}'] 174 175 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 176 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 177 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 178 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 179 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 180 181 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 182 Valid Value pel_serial_number ['${redfish_serial_number}'] 183 184 185Verify Host Off State From PEL 186 [Documentation] Verify Host off state from PEL. 187 [Tags] Verify_Host_Off_State_From_PEL 188 189 Redfish Power Off stack_mode=skip 190 Create Test PEL Log 191 192 ${pel_ids}= Get PEL Log Via BMC CLI 193 ${id}= Get From List ${pel_ids} -1 194 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 195 196 Valid Value pel_host_state ['Off'] 197 198 199Verify Host On State From PEL 200 [Documentation] Verify Host on state from PEL. 201 [Tags] Verify_Host_On_State_From_PEL 202 203 Redfish Power On stack_mode=skip 204 Redfish Purge Event Log 205 206 Create Test PEL Log 207 208 ${pel_ids}= Get PEL Log Via BMC CLI 209 ${id}= Get From List ${pel_ids} -1 210 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 211 212 Valid Value pel_host_state ['Running'] 213 214 215Verify BMC Version From PEL 216 [Documentation] Verify BMC Version from PEL. 217 [Tags] Verify_BMC_Version_From_PEL 218 219 Create Test PEL Log 220 221 ${pel_ids}= Fetch All Pel Ids Based On Error Message test error 222 ${pel_bmc_version}= Get PEL Field Value ${pel_ids[0]} User Data FW Version ID 223 224 ${bmc_version}= Get BMC Version 225 Valid Value bmc_version ['${bmc_version}'] 226 227 228Verify PEL Log After Host Poweron 229 [Documentation] Verify PEL log generation while booting host. 230 [Tags] Verify_PEL_Log_After_Host_Poweron 231 232 Redfish Power Off stack_mode=skip 233 Redfish Purge Event Log 234 Redfish Power On stack_mode=skip 235 236 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 237 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 238 239 # Get BMC created non-infomational error. 240 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 241 242 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 243 244 245Verify BMC Event Log ID 246 [Documentation] Verify BMC Event Log ID from PEL. 247 [Tags] Verify_BMC_Event_Log_ID 248 249 Redfish Purge Event Log 250 Create Test PEL Log 251 252 ${pel_ids}= Get PEL Log Via BMC CLI 253 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 254 255 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 256 # [Private Header]: 257 # [Created at]: 08/24/1928 12:04:06 258 # [Created by]: 0x584D 259 # [Sub-section type]: 0 260 # [Entry Id]: 0x50000BB7 261 # [Platform Log Id]: 0x8200061D 262 # [CSSVER]: 263 # [Section Version]: 1 264 # [Creator Subsystem]: PHYP 265 # [BMC Event Log Id]: 341 <---- BMC event log id value 266 # [Committed at]: 03/25/1920 12:06:22 267 268 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries 269 270 # Example of redfish_event_logs output: 271 # redfish_event_logs: 272 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 273 # [Name]: System Event Log Entries 274 # [Members@odata.count]: 1 275 # [@odata.type]: #LogEntryCollection.LogEntryCollection 276 # [Description]: Collection of System Event Log Entries 277 # [Members]: 278 # [0]: 279 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 280 # [Name]: System Event Log Entry 281 # [Severity]: Critical 282 # [EntryType]: Event 283 # [Created]: 2020-04-02T07:25:13+00:00 284 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 285 # [Id]: 235 <----- Event log ID 286 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 287 288 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 289 290 291Verify FRU Callout 292 [Documentation] Verify FRU callout entries from PEL log. 293 [Tags] Verify_FRU_Callout 294 295 # Power off the system to avoid unnecessary error logs during test. 296 Redfish Power Off stack_mode=skip 297 Create Test PEL Log FRU Callout 298 299 ${pel_ids}= Get PEL Log Via BMC CLI 300 ${id}= Get From List ${pel_ids} -1 301 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 302 303 # Example of PEL Callout Section from "peltool -i <id>" command. 304 # [Callouts]: 305 # [0]: 306 # [FRU Type]: Normal Hardware FRU 307 # [Priority]: Mandatory, replace all with this type as a unit 308 # [Location Code]: U78DA.ND1.1234567-P0 309 # [Part Number]: F191014 310 # [CCIN]: 2E2D 311 # [Serial Number]: YL2E2D010000 312 # [Callout Count]: 1 313 314 Valid Value pel_callout_section['Callout Count'] ['1'] 315 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 316 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 317 318 # Verify Location Code field of PEL callout with motherboard's Location Code. 319 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 320 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 321 ... ${busctl_output[0].split('"')[1].strip('"')} 322 323 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 324 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 325 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 326 327 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 328 ${busctl_output}= BMC Execute Command 329 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 330 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 331 ... ${busctl_output[0].split('"')[1].strip('"')} 332 333 334Verify Procedure And Symbolic FRU Callout 335 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 336 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 337 338 Create Test PEL Log Procedure And Symbolic FRU Callout 339 340 ${pel_ids}= Get PEL Log Via BMC CLI 341 ${id}= Get From List ${pel_ids} -1 342 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 343 344 # Example of PEL Callout Section from "peltool -i <id>" command. 345 # [Callouts]: 346 # [0]: 347 # [Priority]: Mandatory, replace all with this type as a unit 348 # [Procedure Number]: BMCSP02 349 # [FRU Type]: Maintenance Procedure Required 350 # [1]: 351 # [Priority]: Medium Priority 352 # [Part Number]: SVCDOCS 353 # [FRU Type]: Symbolic FRU 354 # [Callout Count]: 2 355 356 Valid Value pel_callout_section['Callout Count'] ['2'] 357 358 # Verify procedural callout info. 359 360 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 361 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 362 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 363 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 364 365 # Verify procedural callout info. 366 367 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 368 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 369 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 370 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 371 372 373Verify PEL Log Entry For Event Log 374 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 375 [Tags] Verify_PEL_Log_Entry_For_Event_Log 376 377 Redfish Purge Event Log 378 379 # Create an unrecoverable error log. 380 Create Test PEL Log Unrecoverable Error 381 382 ${elog_entry}= Get Event Logs 383 # Example of Redfish event logs: 384 # elog_entry: 385 # [0]: 386 # [Message]: BD802003 event in subsystem: Platform Firmware 387 # [Created]: 2023-05-08T10:19:30.412+00:00 388 # [Id]: 1 389 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 390 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 391 # [EntryType]: Event 392 # [Severity]: Critical 393 # [Name]: System Event Log Entry 394 395 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"].split('.')[0]} epoch 396 397 ${pel_records}= Peltool -l 398 # Example output from 'Peltool -l': 399 # pel_records: 400 # [0x50000023]: 401 # [SRC]: BD802003 402 # [CreatorID]: BMC 403 # [Message]: This is a test error 404 # [CompID]: 0x2000 405 # [PLID]: 0x500053D9 406 # [Commit Time]: 04/20/2020 01:55:22 407 # [Subsystem]: Platform Firmware 408 # [Sev]: Unrecoverable Error 409 410 ${ids}= Get Dictionary Keys ${pel_records} 411 ${id}= Get From List ${ids} 0 412 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 413 ... date_format=%m/%d/%Y %H:%M:%S 414 415 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 416 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 417 Should Contain ${elog_entry[0]['Message']} 418 ... ${pel_records['${id}']['SRC']} ignore_case=True 419 420 Should Be Equal ${redfish_log_time} ${pel_log_time} 421 422 423Verify Delete All PEL 424 [Documentation] Verify deleting all PEL logs. 425 [Tags] Verify_Delete_All_PEL 426 427 Create Test PEL Log 428 Create Test PEL Log 429 Peltool --delete-all parse_json=False 430 431 ${pel_ids}= Get PEL Log Via BMC CLI 432 Should Be Empty ${pel_ids} 433 434 435Verify Informational Error Log 436 [Documentation] Create an informational error log and verify. 437 [Tags] Verify_Informational_Error_Log 438 439 Redfish Purge Event Log 440 # Create an informational error log. 441 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 442 ${pel_records}= Peltool -lfh 443 444 # An example of information error log data: 445 # { 446 # "0x500006A0": { 447 # "SRC": "BD8D1002", 448 # "Message": "An application had an internal failure", 449 # "PLID": "0x500006A0", 450 # "CreatorID": "BMC", 451 # "Subsystem": "BMC Firmware", 452 # "Commit Time": "10/14/2020 11:41:38", 453 # "Sev": "Informational Event", 454 # "CompID": "0x1000" 455 # } 456 # } 457 458 ${ids}= Get Dictionary Keys ${pel_records} 459 ${id}= Get From List ${ids} 0 460 Should Contain ${pel_records['${id}']['Sev']} Informational 461 462 463Verify Predictable Error Log 464 [Documentation] Create a predictive error and verify. 465 [Tags] Verify_Predictable_Error_Log 466 467 # Create a predictable error log. 468 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 469 ${pel_records}= Peltool -l 470 471 # An example of predictive error log data: 472 # { 473 # "0x5000069E": { 474 # "SRC": "BD8D1002", 475 # "Message": "An application had an internal failure", 476 # "PLID": "0x5000069E", 477 # "CreatorID": "BMC", 478 # "Subsystem": "BMC Firmware", 479 # "Commit Time": "10/14/2020 11:40:07", 480 # "Sev": "Predictive Error", 481 # "CompID": "0x1000" 482 # } 483 # } 484 485 ${pel_ids}= Get PEL Log Via BMC CLI 486 ${id}= Get From List ${pel_ids} -1 487 Should Contain ${pel_records['${id}']['Sev']} Predictive 488 489 490Verify Unrecoverable Error Log 491 [Documentation] Create an unrecoverable error and verify. 492 [Tags] Verify_Unrecoverable_Error_Log 493 494 # Create an internal failure error log. 495 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 496 ${pel_records}= Peltool -l 497 498 # An example of unrecoverable error log data: 499 # { 500 # "0x50000CC5": { 501 # "SRC": "BD8D1002", 502 # "Message": "An application had an internal failure", 503 # "PLID": "0x50000CC5", 504 # "CreatorID": "BMC", 505 # "Subsystem": "BMC Firmware", 506 # "Commit Time": "04/01/2020 16:44:55", 507 # "Sev": "Unrecoverable Error", 508 # "CompID": "0x1000" 509 # } 510 # } 511 512 ${pel_ids}= Get PEL Log Via BMC CLI 513 ${id}= Get From List ${pel_ids} -1 514 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 515 516 517Verify Error Logging Rotation Policy 518 [Documentation] Verify error logging rotation policy. 519 [Tags] Verify_Error_Logging_Rotation_Policy 520 [Template] Error Logging Rotation Policy 521 522 # Error logs to be created % of total logging space when error 523 # log exceeds max limit. 524 Informational BMC 3000 15 525 Predictive BMC 3000 30 526 Unrecoverable BMC 3000 30 527 Informational BMC 1500, Predictive BMC 1500 45 528 Informational BMC 1500, Unrecoverable BMC 1500 45 529 Unrecoverable BMC 1500, Predictive BMC 1500 30 530 531 532Verify Error Logging Rotation Policy With All Types Of Errors 533 [Documentation] Verify error logging rotation policy with all types of errors. 534 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 535 [Template] Error Logging Rotation Policy 536 537 # Error logs to be created % of total logging space when error 538 # log exceeds max limit. 539 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 540 541 542Verify Error Logging Rotation Policy With HOST Error Logs 543 [Documentation] Verify error logging rotation policy for non bmc error logs. 544 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 545 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 546 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 547 ... /tmp/FILE_HOST_INFORMATIONAL 548 [Template] Error Logging Rotation Policy 549 550 # Error logs to be created % of total logging space when error 551 # log exceeds max limit. 552 Informational HOST 3000 15 553 Unrecoverable HOST 3000 30 554 Informational HOST 1500, Informational BMC 1500 30 555 Informational HOST 1500, Unrecoverable BMC 1500 45 556 Unrecoverable HOST 1500, Informational BMC 1500 45 557 Unrecoverable HOST 1500, Predictive BMC 1500 60 558 559 560Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 561 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 562 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 563 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 564 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 565 [Template] Error Logging Rotation Policy 566 567 # Error logs to be created % of total logging space when error 568 # log exceeds max limit. 569 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 570 571 572Verify Old Logs Are Deleted When Count Crosses Max 573 [Documentation] Verify that when the count crosses max, older logs are deleted. 574 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 575 576 Redfish Purge Event Log 577 # One information log is created after purging error log and additional 2999 are created. 578 # In total, 3000 error logs are available. 579 FOR ${count} IN RANGE 1 ${3000} 580 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 581 END 582 583 # Retrieve the IDs of the logs. 584 ${pel_ids}= Get PEL Log Via BMC CLI 585 ${1st_id}= Get From List ${pel_ids} 0 586 ${3000th_id}= Get From List ${pel_ids} 2999 587 588 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 589 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 590 591 # Wait few seconds for error logs rotation to complete. 592 Sleep 10s 593 594 # Now verify that first log is no more available but the 3000th is available. 595 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 596 Should Be True '${status}' == 'FAIL' 597 Should Contain ${output} PEL not found 598 599 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 600 Should Be True '${status}' == 'PASS' 601 Should Not Contain ${output} PEL not found 602 603 604Verify Reverse Order Of PEL Logs 605 [Documentation] Verify PEL command to output PEL logs in reverse order. 606 [Tags] Verify_Reverse_Order_Of_PEL_Logs 607 608 Redfish Purge Event Log 609 610 # Below commands create unrecoverable error log at first and then the predictable error. 611 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 612 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 613 614 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 615 ${pel_records}= peltool -lr 616 617 # It is found that, variables like dictionary always keep items in sorted order that makes 618 # this verification not possible, hence json is used to keep the items original order. 619 ${pel_records}= Convert To String ${pel_records} 620 ${json_string}= Replace String ${pel_records} ' " 621 ${json_object}= Evaluate json.loads('''${json_string}''') json 622 623 ${list}= Convert To List ${json_object} 624 625 ${id1}= Get From List ${list} 0 626 ${id1}= Convert To Integer ${id1} 627 ${id2}= Get From List ${list} 1 628 ${id2}= Convert To Integer ${id2} 629 630 Should Be True ${id1} > ${id2} 631 632 633Verify Total PEL Count 634 [Documentation] Verify total PEL count returned by peltool command. 635 [Tags] Verify_Total_PEL_Count 636 637 # Initially remove all logs. 638 Redfish Purge Event Log 639 640 # Generate a random number between 1-20. 641 ${random}= Evaluate random.randint(1, 20) modules=random 642 643 # Generate predictive error log multiple times. 644 FOR ${count} IN RANGE 0 ${random} 645 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 646 END 647 648 # Check PEL log count via peltool command and compare it with actual generated log count. 649 ${pel_records}= peltool -n 650 651 Should Be Equal ${pel_records['Number of PELs found']} ${random} 652 653 654Verify Listing Information Error 655 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 656 [Tags] Verify_Listing_Information_Error 657 658 # Initially remove all logs. 659 Redfish Purge Event Log 660 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 661 662 # Generate informational logs and verify that it would not get listed by peltool's list command. 663 ${pel_records}= peltool -l 664 ${ids}= Get Dictionary Keys ${pel_records} 665 Should Be Empty ${ids} 666 667 # Verify that information logs get listed using peltool's list command with -lfh option. 668 ${pel_records}= peltool -lfh 669 ${ids}= Get Dictionary Keys ${pel_records} 670 Should Not Be Empty ${ids} 671 ${id}= Get From List ${ids} 0 672 Should Contain ${pel_records['${id}']['Sev']} Informational 673 674 675Verify PEL Delete 676 [Documentation] Verify that peltool command can delete PEL log based on id. 677 [Tags] Verify_PEL_Delete 678 679 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 680 ${pel_ids}= Get PEL Log Via BMC CLI 681 ${id}= Get From List ${pel_ids} -1 682 Peltool -d ${id} parse_json=False 683 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 684 685 686Verify Mandatory Fields For Predictive Error 687 [Documentation] Verify mandatory fields of predictive errors from pel information. 688 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 689 690 # Inject predictive error. 691 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 692 693 ${pel_ids}= Get PEL Log Via BMC CLI 694 ${pel_id}= Get From List ${pel_ids} -1 695 ${pel_output}= Peltool -i ${pel_id} 696 # Get all fields in predictive error log. 697 ${pel_sections}= Get Dictionary Keys ${pel_output} 698 699 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fields} 700 701 702Verify Mandatory Fields For Informational Error 703 [Documentation] Verify mandatory fields of informational error from pel information. 704 [Tags] Verify_Mandatory_Fields_For_Informational_Error 705 706 # Inject informational error. 707 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 708 ${pel_records}= Peltool -lfh 709 710 ${ids}= Get Dictionary Keys ${pel_records} 711 ${pel_id}= Get From List ${ids} -1 712 ${pel_output}= Peltool -i ${pel_id} 713 714 # Get all fields in the informational error log. 715 ${pel_sections}= Get Dictionary Keys ${pel_output} 716 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 717 ${contains}= Evaluate "${section}" in "${pel_sections}" 718 Should Be True ${contains} 719 END 720 721 722Verify PEL Log Offloaded To Host 723 [Documentation] Verify host transmission state of offloaded PEL logs. 724 [Tags] Verify_PEL_Log_Offloaded_To_Host 725 [Template] Verify PEL Transmission To Host 726 727 # error_type host_state expected_transmission_state 728 predictive_error On Acked 729 unrecoverable_error On Acked 730 informational_error On Acked 731 732 733Verify PEL Log Not Offloaded To Host 734 [Documentation] Verify host transmission state of not offloaded PEL logs. 735 [Tags] Verify_PEL_Log_Not_Offloaded_To_Host 736 [Template] Verify PEL Transmission To Host 737 738 # error_type host_state expected_transmission_state 739 predictive_error Off Not Sent 740 unrecoverable_error Off Not Sent 741 informational_error Off Not Sent 742 743 744*** Keywords *** 745 746Error Logging Rotation Policy 747 [Documentation] Verify that when maximum log limit is reached, given error logging type 748 ... are deleted when reached their max allocated space. 749 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 750 751 # Description of argument(s): 752 # error_log Error logs to be created (E.g. Informational BMC 3000 753 # stands for BMC created 3000 informational error logs). 754 # max_allocated_space_percentage The maximum percentage of disk usage for given error 755 # log type when maximum count/log size is reached. 756 # The maximum error log count is 3000. 757 758 Redfish.Login 759 760 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 761 # in BMC during the test. 762 Redfish Purge Event Log 763 764 @{lists}= Split String ${error_log_type} ,${SPACE} 765 766 ${length}= Get Length ${lists} 767 768 FOR ${list} IN RANGE ${length} 769 @{words}= Split String ${lists}[${list}] ${SPACE} 770 Create Error Log ${words}[0] ${words}[1] ${words}[2] 771 END 772 773 # Create an additional error log to exceed max error logs limit. 774 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 775 776 # Delay for BMC to perform delete older error logs when log limit exceeds. 777 Sleep 10s 778 779 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 780 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 781 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 782 783 ${disk_usage_percentage}= Get Disk Usage For Error Logs 784 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 785 ${percent_diff}= Evaluate abs(${percent_diff}) 786 787 ${eval_value}= Evaluate ${percent_diff} <= 0.5 788 789 ${trimmed_as_expected}= Set Variable If ${disk_usage_percentage} > ${max_allocated_space_percentage} 790 ... ${eval_value} True 791 792 # Check PEL log count via peltool command and compare it with actual generated log count. 793 ${pel_records}= peltool -n 794 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 795 # Number of logs can be 80% of the total logs created after trimming. 796 ${expected_max_record}= Evaluate 3000 * 0.8 797 798 IF ${trimmed_as_expected} == False 799 Should Be True ${no_pel_records} <= ${expected_max_record} 800 END 801 802Create Error Log 803 [Documentation] Create an error log. 804 [Arguments] ${error_severity} ${error_creator} ${count} 805 806 # Description of argument(s): 807 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 808 # error_creator Name of error log's creator(E.g BMC, Host Boot) 809 # count Number of error logs to be generated. 810 811 FOR ${i} IN RANGE 0 ${count} 812 ${cmd}= Set Variable If 813 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 814 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 815 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 816 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 817 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 818 BMC Execute Command ${cmd} 819 END 820 821 822Get Disk Usage For Error Logs 823 [Documentation] Get disk usage percentage for error logs. 824 825 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 826 827 ${usage_output}= Fetch From Left ${usage_output} \/ 828 829 # Convert disk usage unit from KB to MB. 830 ${usage_output}= Evaluate ${usage_output} / 1024 831 832 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 833 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 834 835 RETURN ${usage_percent} 836 837 838Get PEL Log IDs 839 [Documentation] Returns the list of PEL log IDs which contains given field's value. 840 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 841 842 # Description of argument(s): 843 # pel_section The section of PEL (e.g. Private Header, User Header). 844 # pel_field The PEL field (e.g. Event Severity, Event Type). 845 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 846 847 ${pel_ids}= Get PEL Log Via BMC CLI 848 @{pel_id_list}= Create List 849 850 FOR ${id} IN @{pel_ids} 851 ${pel_output}= Peltool -i ${id} 852 # Example of PEL output from "peltool -i <id>" command. 853 # [Private Header]: 854 # [Created at]: 08/24/1928 12:04:06 855 # [Created by]: 0x584D 856 # [Sub-section type]: 0 857 # [Entry Id]: 0x50000BB7 858 # [Platform Log Id]: 0x8200061D 859 # [CSSVER]: 860 # [Section Version]: 1 861 # [Creator Subsystem]: PHYP 862 # [BMC Event Log Id]: 341 863 # [Committed at]: 03/25/1920 12:06:22 864 # [User Header]: 865 # [Log Committed by]: 0x4552 866 # [Action Flags]: 867 # [0]: Report Externally 868 # [Subsystem]: I/O Subsystem 869 # [Event Type]: Miscellaneous, Informational Only 870 # [Sub-section type]: 0 871 # [Event Scope]: Entire Platform 872 # [Event Severity]: Informational Event 873 # [Host Transmission]: Not Sent 874 # [Section Version]: 1 875 876 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 877 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 878 IF '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 879 END 880 Sort List ${pel_id_list} 881 882 RETURN ${pel_id_list} 883 884 885Get PEL Log Via BMC CLI 886 [Documentation] Returns the list of PEL IDs using BMC CLI. 887 888 ${pel_records}= Peltool -l 889 ${ids}= Get Dictionary Keys ${pel_records} 890 Sort List ${ids} 891 892 RETURN ${ids} 893 894 895Get PEL Field Value 896 [Documentation] Returns the value of given PEL's field. 897 [Arguments] ${pel_id} ${pel_section} ${pel_field} 898 899 # Description of argument(s): 900 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 901 # pel_section The section of PEL (e.g. Private Header, User Header) 902 # pel_field The PEL field (e.g. Event Severity, Event Type). 903 904 ${pel_output}= Peltool -i ${pel_id} 905 906 # Example of PEL output from "peltool -i <id>" command. 907 # [Private Header]: 908 # [Created at]: 08/24/1928 12:04:06 909 # [Created by]: 0x584D 910 # [Sub-section type]: 0 911 # [Entry Id]: 0x50000BB7 912 # [Platform Log Id]: 0x8200061D 913 # [CSSVER]: 914 # [Section Version]: 1 915 # [Creator Subsystem]: PHYP 916 # [BMC Event Log Id]: 341 917 # [Committed at]: 03/25/1920 12:06:22 918 # [User Header]: 919 # [Log Committed by]: 0x4552 920 # [Action Flags]: 921 # [0]: Report Externally 922 # [Subsystem]: I/O Subsystem 923 # [Event Type]: Miscellaneous, Informational Only 924 # [Sub-section type]: 0 925 # [Event Scope]: Entire Platform 926 # [Event Severity]: Informational Event 927 # [Host Transmission]: Not Sent 928 # [Section Version]: 1 929 930 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 931 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 932 933 RETURN ${pel_field_output} 934 935 936Verify PEL Transmission To Host 937 [Documentation] Inject PEL log of given type and verify its host transmission state. 938 [Arguments] ${error_type} ${host_state} ${expected_transmission_state} 939 940 # Description of argument(s): 941 # error_type Type of error log to be injected. 942 # host_state Host state which is required before error log injection. 943 # expected_transmission_state Expected host transmission state of PEL log. 944 945 # Get system in required state. 946 IF '${host_state}' == 'Off' 947 Redfish Power Off stack_mode=skip 948 ELSE IF '${host_state}' == 'On' 949 Redfish Power On stack_mode=skip 950 Wait For Host To Ping ${OS_HOST} 5 min 10 sec 951 END 952 953 Redfish Purge Event Log 954 955 # Inject required error log. 956 IF "${error_type}" == "informational_error" 957 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 958 ELSE IF "${error_type}" == "unrecoverable_error" 959 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 960 ELSE IF "${error_type}" == "predictive_error" 961 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 962 END 963 964 ${pel_records}= Get Pel Data From Bmc True True 965 ${ids}= Get Dictionary Keys ${pel_records} 966 ${pel_id}= Get From List ${ids} -1 967 968 # Check host transmission state for the cases where PEL is 969 # expected to be offloaded to HOST. 970 IF "${expected_transmission_state}" == "Acked" 971 Wait Until Keyword Succeeds 15 min 10 sec 972 ... Check If PEL Transmission State Is Expected ${pel_id} Acked 973 END 974 975 # Adding delay before checking host transmission for the cases where PEL is 976 # not expected to be offloaded to HOST. 977 IF "${expected_transmission_state}" == "Not sent" 978 Sleep 120s 979 Check If PEL Transmission State Is Expected ${pel_id} Not sent 980 END 981 982Check If PEL Transmission State Is Expected 983 [Documentation] Check if PEL's host transmission state is matching expected state. 984 [Arguments] ${pel_id} ${expected_transmission_state} 985 986 # Description of argument(s): 987 # expected_transmission_state Expected transmission state of PEL log. 988 989 # Check the transmission details in the error log. 990 ${host_transmission}= Get PEL Field Value ${pel_id} User Header Host Transmission 991 Should Be Equal ${host_transmission} ${expected_transmission_state} 992