1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14Force Tags Bmc_Pel 15 16*** Variables *** 17 18@{mandatory_pel_fields} Private Header User Header Primary SRC 19... Extended User Header Failing MTMS 20@{mandatory_Predictive_pel_fields} Private Header User Header Primary SRC 21... Extended User Header Failing MTMS User Data 22@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 23... Extended User Header User Data 24 25*** Test Cases *** 26 27Create Test PEL Log And Verify 28 [Documentation] Create PEL log using busctl command and verify via peltool. 29 [Tags] Create_Test_PEL_Log_And_Verify 30 31 Redfish Purge Event Log 32 Create Test PEL Log 33 ${pel_id}= Get PEL Log Via BMC CLI 34 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 35 36 37Verify PEL Log Details 38 [Documentation] Verify PEL log details via peltool. 39 [Tags] Verify_PEL_Log_Details 40 41 Redfish Purge Event Log 42 43 ${bmc_time1}= CLI Get BMC DateTime 44 Create Test PEL Log 45 ${bmc_time2}= CLI Get BMC DateTime 46 47 ${pel_records}= Peltool -l 48 49 # Example output from 'Peltool -l': 50 # pel_records: 51 # [0x50000012]: 52 # [CreatorID]: BMC 53 # [CompID]: 0x1000 54 # [PLID]: 0x50000012 55 # [Subsystem]: BMC Firmware 56 # [Message]: An application had an internal failure 57 # [SRC]: BD8D1002 58 # [Commit Time]: 03/02/2020 09:35:15 59 # [Sev]: Unrecoverable Error 60 61 ${ids}= Get Dictionary Keys ${pel_records} 62 ${id}= Get From List ${ids} 0 63 64 @{pel_fields}= Create List CreatorID Subsystem Message Sev 65 FOR ${field} IN @{pel_fields} 66 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 67 END 68 69 Valid Value pel_records['${id}']['PLID'] ['${id}'] 70 71 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 72 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 73 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 74 75 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 76 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 77 78 # Convert BMC and PEL time to epoch time before comparing. 79 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 80 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 81 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 82 83 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 84 85 86Verify Mandatory Sections Of Error Log PEL 87 [Documentation] Verify mandatory sections of error log PEL. 88 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 89 90 Create Test PEL Log 91 92 ${pel_ids}= Get PEL Log Via BMC CLI 93 ${pel_id}= Get From List ${pel_ids} -1 94 ${pel_output}= Peltool -i ${pel_id} 95 ${pel_sections}= Get Dictionary Keys ${pel_output} 96 97 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fields} 98 99 100Verify PEL Log Persistence After BMC Reboot 101 [Documentation] Verify PEL log persistence after BMC reboot. 102 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 103 104 Create Test PEL Log 105 ${pel_before_reboot}= Get PEL Log Via BMC CLI 106 107 Redfish OBMC Reboot (off) 108 ${pel_after_reboot}= Get PEL Log Via BMC CLI 109 110 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 111 112 113Verify PEL ID Numbering 114 [Documentation] Verify PEL ID numbering. 115 [Tags] Verify_PEL_ID_Numbering 116 117 Redfish Purge Event Log 118 Create Test PEL Log 119 Create Test PEL Log 120 121 ${pel_ids}= Get PEL Log Via BMC CLI 122 123 # Example of PEL IDs from PEL logs. 124 # [0x50000012]: <--- First PEL ID 125 # [CreatorID]: BMC 126 # [CompID]: 0x1000 127 # [PLID]: 0x50000012 128 # [Subsystem]: BMC Firmware 129 # [Message]: An application had an internal failure 130 # [SRC]: BD8D1002 131 # [Commit Time]: 03/02/2020 09:35:15 132 # [Sev]: Unrecoverable Error 133 # 134 # [0x50000013]: <--- Second PEL ID 135 # [CreatorID]: BMC 136 # [CompID]: 0x1000 137 # [PLID]: 0x50000013 138 # [Subsystem]: BMC Firmware 139 # [Message]: An application had an internal failure 140 # [SRC]: BD8D1002 141 # [Commit Time]: 03/02/2020 09:35:15 142 # [Sev]: Unrecoverable Error 143 144 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 145 146Verify Machine Type Model And Serial Number 147 [Documentation] Verify machine type model and serial number from PEL. 148 [Tags] Verify_Machine_Type_Model_And_Serial_Number 149 150 Create Test PEL Log 151 152 ${pel_ids}= Get PEL Log Via BMC CLI 153 ${id}= Get From List ${pel_ids} -1 154 155 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 156 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 157 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 158 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 159 160 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 161 # [Failing MTMS]: 162 # [Created by]: 0x2000 163 # [Machine Type Model]: 1234-ABC <---- Machine type 164 # [Section Version]: 1 165 # [Serial Number]: ABCDEFG <---- Serial number 166 # [Sub-section type]: 0 167 168 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/${SYSTEM_ID}/ Model 169 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 170 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/${SYSTEM_ID}/ SerialNumber 171 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 172 173 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 174 Valid Value pel_serial_number ['${redfish_serial_number}'] 175 176 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 177 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 178 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 179 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 180 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 181 182 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 183 Valid Value pel_serial_number ['${redfish_serial_number}'] 184 185 186Verify Host Off State From PEL 187 [Documentation] Verify Host off state from PEL. 188 [Tags] Verify_Host_Off_State_From_PEL 189 190 Redfish Power Off stack_mode=skip 191 Create Test PEL Log 192 193 ${pel_ids}= Get PEL Log Via BMC CLI 194 ${id}= Get From List ${pel_ids} -1 195 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 196 197 Valid Value pel_host_state ['Off'] 198 199 200Verify Host On State From PEL 201 [Documentation] Verify Host on state from PEL. 202 [Tags] Verify_Host_On_State_From_PEL 203 204 Redfish Power On stack_mode=skip 205 Redfish Purge Event Log 206 207 Create Test PEL Log 208 209 ${pel_ids}= Get PEL Log Via BMC CLI 210 ${id}= Get From List ${pel_ids} -1 211 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 212 213 Valid Value pel_host_state ['Running'] 214 215 216Verify BMC Version From PEL 217 [Documentation] Verify BMC Version from PEL. 218 [Tags] Verify_BMC_Version_From_PEL 219 220 Create Test PEL Log 221 222 ${pel_ids}= Get PEL Log Via BMC CLI 223 ${id}= Get From List ${pel_ids} -1 224 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 225 226 ${bmc_version}= Get BMC Version 227 Valid Value bmc_version ['${bmc_version}'] 228 229 230Verify PEL Log After Host Poweron 231 [Documentation] Verify PEL log generation while booting host. 232 [Tags] Verify_PEL_Log_After_Host_Poweron 233 234 Redfish Power Off stack_mode=skip 235 Redfish Purge Event Log 236 Redfish Power On stack_mode=skip 237 238 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 239 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 240 241 # Get BMC created non-infomational error. 242 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 243 244 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 245 246 247Verify BMC Event Log ID 248 [Documentation] Verify BMC Event Log ID from PEL. 249 [Tags] Verify_BMC_Event_Log_ID 250 251 Redfish Purge Event Log 252 Create Test PEL Log 253 254 ${pel_ids}= Get PEL Log Via BMC CLI 255 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 256 257 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 258 # [Private Header]: 259 # [Created at]: 08/24/1928 12:04:06 260 # [Created by]: 0x584D 261 # [Sub-section type]: 0 262 # [Entry Id]: 0x50000BB7 263 # [Platform Log Id]: 0x8200061D 264 # [CSSVER]: 265 # [Section Version]: 1 266 # [Creator Subsystem]: PHYP 267 # [BMC Event Log Id]: 341 <---- BMC event log id value 268 # [Committed at]: 03/25/1920 12:06:22 269 270 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/${SYSTEM_ID}/LogServices/EventLog/Entries 271 272 # Example of redfish_event_logs output: 273 # redfish_event_logs: 274 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 275 # [Name]: System Event Log Entries 276 # [Members@odata.count]: 1 277 # [@odata.type]: #LogEntryCollection.LogEntryCollection 278 # [Description]: Collection of System Event Log Entries 279 # [Members]: 280 # [0]: 281 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 282 # [Name]: System Event Log Entry 283 # [Severity]: Critical 284 # [EntryType]: Event 285 # [Created]: 2020-04-02T07:25:13+00:00 286 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 287 # [Id]: 235 <----- Event log ID 288 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 289 290 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 291 292 293Verify FRU Callout 294 [Documentation] Verify FRU callout entries from PEL log. 295 [Tags] Verify_FRU_Callout 296 297 # Power off the system to avoid unnecessary error logs during test. 298 Redfish Power Off stack_mode=skip 299 Create Test PEL Log FRU Callout 300 301 ${pel_ids}= Get PEL Log Via BMC CLI 302 ${id}= Get From List ${pel_ids} -1 303 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 304 305 # Example of PEL Callout Section from "peltool -i <id>" command. 306 # [Callouts]: 307 # [0]: 308 # [FRU Type]: Normal Hardware FRU 309 # [Priority]: Mandatory, replace all with this type as a unit 310 # [Location Code]: U78DA.ND1.1234567-P0 311 # [Part Number]: F191014 312 # [CCIN]: 2E2D 313 # [Serial Number]: YL2E2D010000 314 # [Callout Count]: 1 315 316 Valid Value pel_callout_section['Callout Count'] ['1'] 317 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 318 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 319 320 # Verify Location Code field of PEL callout with motherboard's Location Code. 321 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 322 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 323 ... ${busctl_output[0].split('"')[1].strip('"')} 324 325 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 326 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 327 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 328 329 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 330 ${busctl_output}= BMC Execute Command 331 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 332 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 333 ... ${busctl_output[0].split('"')[1].strip('"')} 334 335 336Verify Procedure And Symbolic FRU Callout 337 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 338 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 339 340 Create Test PEL Log Procedure And Symbolic FRU Callout 341 342 ${pel_ids}= Get PEL Log Via BMC CLI 343 ${id}= Get From List ${pel_ids} -1 344 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 345 346 # Example of PEL Callout Section from "peltool -i <id>" command. 347 # [Callouts]: 348 # [0]: 349 # [Priority]: Mandatory, replace all with this type as a unit 350 # [Procedure Number]: BMCSP02 351 # [FRU Type]: Maintenance Procedure Required 352 # [1]: 353 # [Priority]: Medium Priority 354 # [Part Number]: SVCDOCS 355 # [FRU Type]: Symbolic FRU 356 # [Callout Count]: 2 357 358 Valid Value pel_callout_section['Callout Count'] ['2'] 359 360 # Verify procedural callout info. 361 362 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 363 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 364 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 365 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 366 367 # Verify procedural callout info. 368 369 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 370 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 371 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 372 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 373 374 375Verify PEL Log Entry For Event Log 376 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 377 [Tags] Verify_PEL_Log_Entry_For_Event_Log 378 379 Redfish Purge Event Log 380 381 # Create an unrecoverable error log. 382 Create Test PEL Log Unrecoverable Error 383 384 ${elog_entry}= Get Event Logs 385 # Example of Redfish event logs: 386 # elog_entry: 387 # [0]: 388 # [Message]: BD802003 event in subsystem: Platform Firmware 389 # [Created]: 2023-05-08T10:19:30.412+00:00 390 # [Id]: 1 391 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 392 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 393 # [EntryType]: Event 394 # [Severity]: Critical 395 # [Name]: System Event Log Entry 396 397 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"].split('.')[0]} epoch 398 399 ${pel_records}= Peltool -l 400 # Example output from 'Peltool -l': 401 # pel_records: 402 # [0x50000023]: 403 # [SRC]: BD802003 404 # [CreatorID]: BMC 405 # [Message]: This is a test error 406 # [CompID]: 0x2000 407 # [PLID]: 0x500053D9 408 # [Commit Time]: 04/20/2020 01:55:22 409 # [Subsystem]: Platform Firmware 410 # [Sev]: Unrecoverable Error 411 412 ${ids}= Get Dictionary Keys ${pel_records} 413 ${id}= Get From List ${ids} 0 414 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 415 ... date_format=%m/%d/%Y %H:%M:%S 416 417 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 418 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 419 Should Contain ${elog_entry[0]['Message']} 420 ... ${pel_records['${id}']['SRC']} ignore_case=True 421 422 Should Be Equal ${redfish_log_time} ${pel_log_time} 423 424 425Verify Delete All PEL 426 [Documentation] Verify deleting all PEL logs. 427 [Tags] Verify_Delete_All_PEL 428 429 Create Test PEL Log 430 Create Test PEL Log 431 Peltool --delete-all False 432 433 ${pel_ids}= Get PEL Log Via BMC CLI 434 Should Be Empty ${pel_ids} 435 436 437Verify Informational Error Log 438 [Documentation] Create an informational error log and verify. 439 [Tags] Verify_Informational_Error_Log 440 441 Redfish Purge Event Log 442 # Create an informational error log. 443 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 444 ${pel_records}= Peltool -lfh 445 446 # An example of information error log data: 447 # { 448 # "0x500006A0": { 449 # "SRC": "BD8D1002", 450 # "Message": "An application had an internal failure", 451 # "PLID": "0x500006A0", 452 # "CreatorID": "BMC", 453 # "Subsystem": "BMC Firmware", 454 # "Commit Time": "10/14/2020 11:41:38", 455 # "Sev": "Informational Event", 456 # "CompID": "0x1000" 457 # } 458 # } 459 460 ${ids}= Get Dictionary Keys ${pel_records} 461 ${id}= Get From List ${ids} 0 462 Should Contain ${pel_records['${id}']['Sev']} Informational 463 464 465Verify Predictable Error Log 466 [Documentation] Create a predictive error and verify. 467 [Tags] Verify_Predictable_Error_Log 468 469 # Create a predictable error log. 470 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 471 ${pel_records}= Peltool -l 472 473 # An example of predictive error log data: 474 # { 475 # "0x5000069E": { 476 # "SRC": "BD8D1002", 477 # "Message": "An application had an internal failure", 478 # "PLID": "0x5000069E", 479 # "CreatorID": "BMC", 480 # "Subsystem": "BMC Firmware", 481 # "Commit Time": "10/14/2020 11:40:07", 482 # "Sev": "Predictive Error", 483 # "CompID": "0x1000" 484 # } 485 # } 486 487 ${pel_ids}= Get PEL Log Via BMC CLI 488 ${id}= Get From List ${pel_ids} -1 489 Should Contain ${pel_records['${id}']['Sev']} Predictive 490 491 492Verify Unrecoverable Error Log 493 [Documentation] Create an unrecoverable error and verify. 494 [Tags] Verify_Unrecoverable_Error_Log 495 496 # Create an internal failure error log. 497 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 498 ${pel_records}= Peltool -l 499 500 # An example of unrecoverable error log data: 501 # { 502 # "0x50000CC5": { 503 # "SRC": "BD8D1002", 504 # "Message": "An application had an internal failure", 505 # "PLID": "0x50000CC5", 506 # "CreatorID": "BMC", 507 # "Subsystem": "BMC Firmware", 508 # "Commit Time": "04/01/2020 16:44:55", 509 # "Sev": "Unrecoverable Error", 510 # "CompID": "0x1000" 511 # } 512 # } 513 514 ${pel_ids}= Get PEL Log Via BMC CLI 515 ${id}= Get From List ${pel_ids} -1 516 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 517 518 519Verify Error Logging Rotation Policy 520 [Documentation] Verify error logging rotation policy. 521 [Tags] Verify_Error_Logging_Rotation_Policy 522 [Template] Error Logging Rotation Policy 523 524 # Error logs to be created % of total logging space when error 525 # log exceeds max limit. 526 Informational BMC 3000 15 527 Predictive BMC 3000 30 528 Unrecoverable BMC 3000 30 529 Informational BMC 1500, Predictive BMC 1500 45 530 Informational BMC 1500, Unrecoverable BMC 1500 45 531 Unrecoverable BMC 1500, Predictive BMC 1500 30 532 533 534Verify Error Logging Rotation Policy With All Types Of Errors 535 [Documentation] Verify error logging rotation policy with all types of errors. 536 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 537 [Template] Error Logging Rotation Policy 538 539 # Error logs to be created % of total logging space when error 540 # log exceeds max limit. 541 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 542 543 544Verify Error Logging Rotation Policy With HOST Error Logs 545 [Documentation] Verify error logging rotation policy for non bmc error logs. 546 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 547 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 548 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 549 ... /tmp/FILE_HOST_INFORMATIONAL 550 [Template] Error Logging Rotation Policy 551 552 # Error logs to be created % of total logging space when error 553 # log exceeds max limit. 554 Informational HOST 3000 15 555 Unrecoverable HOST 3000 30 556 Informational HOST 1500, Informational BMC 1500 30 557 Informational HOST 1500, Unrecoverable BMC 1500 45 558 Unrecoverable HOST 1500, Informational BMC 1500 45 559 Unrecoverable HOST 1500, Predictive BMC 1500 60 560 561 562Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 563 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 564 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 565 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 566 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 567 [Template] Error Logging Rotation Policy 568 569 # Error logs to be created % of total logging space when error 570 # log exceeds max limit. 571 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 572 573 574Verify Old Logs Are Deleted When Count Crosses Max 575 [Documentation] Verify that when the count crosses max, older logs are deleted. 576 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 577 578 Redfish Purge Event Log 579 # One information log is created after purging error log and additional 2999 are created. 580 # In total, 3000 error logs are available. 581 FOR ${count} IN RANGE 1 ${3000} 582 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 583 END 584 585 # Retrieve the IDs of the logs. 586 ${pel_ids}= Get PEL Log Via BMC CLI 587 ${1st_id}= Get From List ${pel_ids} 0 588 ${3000th_id}= Get From List ${pel_ids} 2999 589 590 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 591 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 592 593 # Wait few seconds for error logs rotation to complete. 594 Sleep 10s 595 596 # Now verify that first log is no more available but the 3000th is available. 597 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 598 Should Be True '${status}' == 'FAIL' 599 Should Contain ${output} PEL not found 600 601 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 602 Should Be True '${status}' == 'PASS' 603 Should Not Contain ${output} PEL not found 604 605 606Verify Reverse Order Of PEL Logs 607 [Documentation] Verify PEL command to output PEL logs in reverse order. 608 [Tags] Verify_Reverse_Order_Of_PEL_Logs 609 610 Redfish Purge Event Log 611 612 # Below commands create unrecoverable error log at first and then the predictable error. 613 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 614 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 615 616 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 617 ${pel_records}= peltool -lr 618 619 # It is found that, variables like dictionary always keep items in sorted order that makes 620 # this verification not possible, hence json is used to keep the items original order. 621 ${pel_records}= Convert To String ${pel_records} 622 ${json_string}= Replace String ${pel_records} ' " 623 ${json_object}= Evaluate json.loads('''${json_string}''') json 624 625 ${list}= Convert To List ${json_object} 626 627 ${id1}= Get From List ${list} 0 628 ${id1}= Convert To Integer ${id1} 629 ${id2}= Get From List ${list} 1 630 ${id2}= Convert To Integer ${id2} 631 632 Should Be True ${id1} > ${id2} 633 634 635Verify Total PEL Count 636 [Documentation] Verify total PEL count returned by peltool command. 637 [Tags] Verify_Total_PEL_Count 638 639 # Initially remove all logs. 640 Redfish Purge Event Log 641 642 # Generate a random number between 1-20. 643 ${random}= Evaluate random.randint(1, 20) modules=random 644 645 # Generate predictive error log multiple times. 646 FOR ${count} IN RANGE 0 ${random} 647 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 648 END 649 650 # Check PEL log count via peltool command and compare it with actual generated log count. 651 ${pel_records}= peltool -n 652 653 Should Be Equal ${pel_records['Number of PELs found']} ${random} 654 655 656Verify Listing Information Error 657 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 658 [Tags] Verify_Listing_Information_Error 659 660 # Initially remove all logs. 661 Redfish Purge Event Log 662 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 663 664 # Generate informational logs and verify that it would not get listed by peltool's list command. 665 ${pel_records}= peltool -l 666 ${ids}= Get Dictionary Keys ${pel_records} 667 Should Be Empty ${ids} 668 669 # Verify that information logs get listed using peltool's list command with -lfh option. 670 ${pel_records}= peltool -lfh 671 ${ids}= Get Dictionary Keys ${pel_records} 672 Should Not Be Empty ${ids} 673 ${id}= Get From List ${ids} 0 674 Should Contain ${pel_records['${id}']['Sev']} Informational 675 676 677Verify PEL Delete 678 [Documentation] Verify that peltool command can delete PEL log based on id. 679 [Tags] Verify_PEL_Delete 680 681 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 682 ${pel_ids}= Get PEL Log Via BMC CLI 683 ${id}= Get From List ${pel_ids} -1 684 Peltool -d ${id} False 685 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 686 687 688Verify Mandatory Fields For Predictive Error 689 [Documentation] Verify mandatory fields of predictive errors from pel information. 690 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 691 692 # Inject predictive error. 693 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 694 695 ${pel_ids}= Get PEL Log Via BMC CLI 696 ${pel_id}= Get From List ${pel_ids} -1 697 ${pel_output}= Peltool -i ${pel_id} 698 # Get all fields in predictive error log. 699 ${pel_sections}= Get Dictionary Keys ${pel_output} 700 701 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fields} 702 703 704Verify Mandatory Fields For Informational Error 705 [Documentation] Verify mandatory fields of informational error from pel information. 706 [Tags] Verify_Mandatory_Fields_For_Informational_Error 707 708 # Inject informational error. 709 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 710 ${pel_records}= Peltool -lfh 711 712 ${ids}= Get Dictionary Keys ${pel_records} 713 ${pel_id}= Get From List ${ids} -1 714 ${pel_output}= Peltool -i ${pel_id} 715 716 # Get all fields in the informational error log. 717 ${pel_sections}= Get Dictionary Keys ${pel_output} 718 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 719 ${contains}= Evaluate "${section}" in "${pel_sections}" 720 Should Be True ${contains} 721 END 722 723 724Verify PEL Log Offloaded To Host 725 [Documentation] Verify host transmission state of offloaded PEL logs. 726 [Tags] Verify_PEL_Log_Offloaded_To_Host 727 [Template] Verify PEL Transmission To Host 728 729 # error_type host_state expected_transmission_state 730 predictive_error On Acked 731 unrecoverable_error On Acked 732 informational_error On Acked 733 734 735Verify PEL Log Not Offloaded To Host 736 [Documentation] Verify host transmission state of not offloaded PEL logs. 737 [Tags] Verify_PEL_Log_Not_Offloaded_To_Host 738 [Template] Verify PEL Transmission To Host 739 740 # error_type host_state expected_transmission_state 741 predictive_error Off Not Sent 742 unrecoverable_error Off Not Sent 743 informational_error Off Not Sent 744 745 746*** Keywords *** 747 748Error Logging Rotation Policy 749 [Documentation] Verify that when maximum log limit is reached, given error logging type 750 ... are deleted when reached their max allocated space. 751 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 752 753 # Description of argument(s): 754 # error_log Error logs to be created (E.g. Informational BMC 3000 755 # stands for BMC created 3000 informational error logs). 756 # max_allocated_space_percentage The maximum percentage of disk usage for given error 757 # log type when maximum count/log size is reached. 758 # The maximum error log count is 3000. 759 760 Redfish.Login 761 762 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 763 # in BMC during the test. 764 Redfish Purge Event Log 765 766 @{lists}= Split String ${error_log_type} ,${SPACE} 767 768 ${length}= Get Length ${lists} 769 770 FOR ${list} IN RANGE ${length} 771 @{words}= Split String ${lists}[${list}] ${SPACE} 772 Create Error Log ${words}[0] ${words}[1] ${words}[2] 773 END 774 775 # Create an additional error log to exceed max error logs limit. 776 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 777 778 # Delay for BMC to perform delete older error logs when log limit exceeds. 779 Sleep 10s 780 781 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 782 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 783 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 784 785 ${disk_usage_percentage}= Get Disk Usage For Error Logs 786 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 787 ${percent_diff}= Evaluate abs(${percent_diff}) 788 789 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 790 ... Evaluate ${percent_diff} <= 0.5 791 ... ELSE 792 ... Set Variable True 793 794 # Check PEL log count via peltool command and compare it with actual generated log count. 795 ${pel_records}= peltool -n 796 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 797 # Number of logs can be 80% of the total logs created after trimming. 798 ${expected_max_record}= Evaluate 3000 * 0.8 799 800 Run Keyword If ${trimmed_as_expected} == False 801 ... Should Be True ${no_pel_records} <= ${expected_max_record} 802 803 804Create Error Log 805 [Documentation] Create an error log. 806 [Arguments] ${error_severity} ${error_creator} ${count} 807 808 # Description of argument(s): 809 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 810 # error_creator Name of error log's creator(E.g BMC, Host Boot) 811 # count Number of error logs to be generated. 812 813 FOR ${i} IN RANGE 0 ${count} 814 ${cmd}= Set Variable If 815 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 816 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 817 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 818 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 819 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 820 BMC Execute Command ${cmd} 821 END 822 823 824Get Disk Usage For Error Logs 825 [Documentation] Get disk usage percentage for error logs. 826 827 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 828 829 ${usage_output}= Fetch From Left ${usage_output} \/ 830 831 # Convert disk usage unit from KB to MB. 832 ${usage_output}= Evaluate ${usage_output} / 1024 833 834 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 835 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 836 837 [return] ${usage_percent} 838 839 840Get PEL Log IDs 841 [Documentation] Returns the list of PEL log IDs which contains given field's value. 842 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 843 844 # Description of argument(s): 845 # pel_section The section of PEL (e.g. Private Header, User Header). 846 # pel_field The PEL field (e.g. Event Severity, Event Type). 847 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 848 849 ${pel_ids}= Get PEL Log Via BMC CLI 850 @{pel_id_list}= Create List 851 852 FOR ${id} IN @{pel_ids} 853 ${pel_output}= Peltool -i ${id} 854 # Example of PEL output from "peltool -i <id>" command. 855 # [Private Header]: 856 # [Created at]: 08/24/1928 12:04:06 857 # [Created by]: 0x584D 858 # [Sub-section type]: 0 859 # [Entry Id]: 0x50000BB7 860 # [Platform Log Id]: 0x8200061D 861 # [CSSVER]: 862 # [Section Version]: 1 863 # [Creator Subsystem]: PHYP 864 # [BMC Event Log Id]: 341 865 # [Committed at]: 03/25/1920 12:06:22 866 # [User Header]: 867 # [Log Committed by]: 0x4552 868 # [Action Flags]: 869 # [0]: Report Externally 870 # [Subsystem]: I/O Subsystem 871 # [Event Type]: Miscellaneous, Informational Only 872 # [Sub-section type]: 0 873 # [Event Scope]: Entire Platform 874 # [Event Severity]: Informational Event 875 # [Host Transmission]: Not Sent 876 # [Section Version]: 1 877 878 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 879 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 880 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 881 END 882 Sort List ${pel_id_list} 883 884 [Return] ${pel_id_list} 885 886 887Get PEL Log Via BMC CLI 888 [Documentation] Returns the list of PEL IDs using BMC CLI. 889 890 ${pel_records}= Peltool -l 891 ${ids}= Get Dictionary Keys ${pel_records} 892 Sort List ${ids} 893 894 [Return] ${ids} 895 896 897Get PEL Field Value 898 [Documentation] Returns the value of given PEL's field. 899 [Arguments] ${pel_id} ${pel_section} ${pel_field} 900 901 # Description of argument(s): 902 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 903 # pel_section The section of PEL (e.g. Private Header, User Header) 904 # pel_field The PEL field (e.g. Event Severity, Event Type). 905 906 ${pel_output}= Peltool -i ${pel_id} 907 908 # Example of PEL output from "peltool -i <id>" command. 909 # [Private Header]: 910 # [Created at]: 08/24/1928 12:04:06 911 # [Created by]: 0x584D 912 # [Sub-section type]: 0 913 # [Entry Id]: 0x50000BB7 914 # [Platform Log Id]: 0x8200061D 915 # [CSSVER]: 916 # [Section Version]: 1 917 # [Creator Subsystem]: PHYP 918 # [BMC Event Log Id]: 341 919 # [Committed at]: 03/25/1920 12:06:22 920 # [User Header]: 921 # [Log Committed by]: 0x4552 922 # [Action Flags]: 923 # [0]: Report Externally 924 # [Subsystem]: I/O Subsystem 925 # [Event Type]: Miscellaneous, Informational Only 926 # [Sub-section type]: 0 927 # [Event Scope]: Entire Platform 928 # [Event Severity]: Informational Event 929 # [Host Transmission]: Not Sent 930 # [Section Version]: 1 931 932 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 933 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 934 935 [Return] ${pel_field_output} 936 937 938Verify PEL Transmission To Host 939 [Documentation] Inject PEL log of given type and verify its host transmission state. 940 [Arguments] ${error_type} ${host_state} ${expected_transmission_state} 941 942 # Description of argument(s): 943 # error_type Type of error log to be injected. 944 # host_state Host state which is required before error log injection. 945 # expected_transmission_state Expected host transmission state of PEL log. 946 947 # Get system in required state. 948 Run Keyword If '${host_state}' == 'Off' 949 ... Redfish Power Off stack_mode=skip 950 ... ELSE IF '${host_state}' == 'On' 951 ... Run Keywords Redfish Power On stack_mode=skip AND 952 ... Wait For Host To Ping ${OS_HOST} 5 min 10 sec 953 954 Redfish Purge Event Log 955 956 # Inject required error log. 957 Run Keyword If "${error_type}" == "informational_error" 958 ... BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 959 ... ELSE IF "${error_type}" == "unrecoverable_error" 960 ... BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 961 ... ELSE IF "${error_type}" == "predictive_error" 962 ... BMC Execute Command ${CMD_PREDICTIVE_ERROR} 963 964 ${pel_records}= Get Pel Data From Bmc True True 965 ${ids}= Get Dictionary Keys ${pel_records} 966 ${pel_id}= Get From List ${ids} -1 967 968 # Check host transmission state for the cases where PEL is 969 # expected to be offloaded to HOST. 970 Run Keyword If "${expected_transmission_state}" == "Acked" 971 ... Wait Until Keyword Succeeds 15 min 10 sec 972 ... Check If PEL Transmission State Is Expected ${pel_id} Acked 973 974 # Adding delay before checking host transmission for the cases where PEL is 975 # not expected to be offloaded to HOST. 976 Run Keyword If "${expected_transmission_state}" == "Not sent" 977 ... Run Keywords Sleep 120s AND 978 ... Check If PEL Transmission State Is Expected ${pel_id} Not sent 979 980 981Check If PEL Transmission State Is Expected 982 [Documentation] Check if PEL's host transmission state is matching expected state. 983 [Arguments] ${pel_id} ${expected_transmission_state} 984 985 # Description of argument(s): 986 # expected_transmission_state Expected transmission state of PEL log. 987 988 # Check the transmission details in the error log. 989 ${host_transmission}= Get PEL Field Value ${pel_id} User Header Host Transmission 990 Should Be Equal ${host_transmission} ${expected_transmission_state} 991