1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 18@{mandatory_Predictive_pel_fileds} Private Header User Header Primary SRC 19... Extended User Header Failing MTMS User Data 20@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 21... Extended User Header User Data 22 23*** Test Cases *** 24 25Create Test PEL Log And Verify 26 [Documentation] Create PEL log using busctl command and verify via peltool. 27 [Tags] Create_Test_PEL_Log_And_Verify 28 29 Redfish Purge Event Log 30 Create Test PEL Log 31 ${pel_id}= Get PEL Log Via BMC CLI 32 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 33 34 35Verify PEL Log Details 36 [Documentation] Verify PEL log details via peltool. 37 [Tags] Verify_PEL_Log_Details 38 39 Redfish Purge Event Log 40 41 ${bmc_time1}= CLI Get BMC DateTime 42 Create Test PEL Log 43 ${bmc_time2}= CLI Get BMC DateTime 44 45 ${pel_records}= Peltool -l 46 47 # Example output from 'Peltool -l': 48 # pel_records: 49 # [0x50000012]: 50 # [CreatorID]: BMC 51 # [CompID]: 0x1000 52 # [PLID]: 0x50000012 53 # [Subsystem]: BMC Firmware 54 # [Message]: An application had an internal failure 55 # [SRC]: BD8D1002 56 # [Commit Time]: 03/02/2020 09:35:15 57 # [Sev]: Unrecoverable Error 58 59 ${ids}= Get Dictionary Keys ${pel_records} 60 ${id}= Get From List ${ids} 0 61 62 @{pel_fields}= Create List CreatorID Subsystem Message Sev 63 FOR ${field} IN @{pel_fields} 64 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 65 END 66 67 Valid Value pel_records['${id}']['PLID'] ['${id}'] 68 69 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 70 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 71 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 72 73 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 74 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 75 76 # Convert BMC and PEL time to epoch time before comparing. 77 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 78 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 79 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 80 81 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 82 83 84Verify Mandatory Sections Of Error Log PEL 85 [Documentation] Verify mandatory sections of error log PEL. 86 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 87 88 Create Test PEL Log 89 90 ${pel_ids}= Get PEL Log Via BMC CLI 91 ${pel_id}= Get From List ${pel_ids} -1 92 ${pel_output}= Peltool -i ${pel_id} 93 ${pel_sections}= Get Dictionary Keys ${pel_output} 94 95 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 96 97 98Verify PEL Log Persistence After BMC Reboot 99 [Documentation] Verify PEL log persistence after BMC reboot. 100 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 101 102 Create Test PEL Log 103 ${pel_before_reboot}= Get PEL Log Via BMC CLI 104 105 Redfish OBMC Reboot (off) 106 ${pel_after_reboot}= Get PEL Log Via BMC CLI 107 108 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 109 110 111Verify PEL ID Numbering 112 [Documentation] Verify PEL ID numbering. 113 [Tags] Verify_PEL_ID_Numbering 114 115 Redfish Purge Event Log 116 Create Test PEL Log 117 Create Test PEL Log 118 119 ${pel_ids}= Get PEL Log Via BMC CLI 120 121 # Example of PEL IDs from PEL logs. 122 # [0x50000012]: <--- First PEL ID 123 # [CreatorID]: BMC 124 # [CompID]: 0x1000 125 # [PLID]: 0x50000012 126 # [Subsystem]: BMC Firmware 127 # [Message]: An application had an internal failure 128 # [SRC]: BD8D1002 129 # [Commit Time]: 03/02/2020 09:35:15 130 # [Sev]: Unrecoverable Error 131 # 132 # [0x50000013]: <--- Second PEL ID 133 # [CreatorID]: BMC 134 # [CompID]: 0x1000 135 # [PLID]: 0x50000013 136 # [Subsystem]: BMC Firmware 137 # [Message]: An application had an internal failure 138 # [SRC]: BD8D1002 139 # [Commit Time]: 03/02/2020 09:35:15 140 # [Sev]: Unrecoverable Error 141 142 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 143 144Verify Machine Type Model And Serial Number 145 [Documentation] Verify machine type model and serial number from PEL. 146 [Tags] Verify_Machine_Type_Model_And_Serial_Number 147 148 Create Test PEL Log 149 150 ${pel_ids}= Get PEL Log Via BMC CLI 151 ${id}= Get From List ${pel_ids} -1 152 153 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 154 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 155 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 156 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 157 158 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 159 # [Failing MTMS]: 160 # [Created by]: 0x2000 161 # [Machine Type Model]: 1234-ABC <---- Machine type 162 # [Section Version]: 1 163 # [Serial Number]: ABCDEFG <---- Serial number 164 # [Sub-section type]: 0 165 166 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 167 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 168 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 169 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 170 171 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 172 Valid Value pel_serial_number ['${redfish_serial_number}'] 173 174 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 175 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 176 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 177 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 178 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 179 180 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 181 Valid Value pel_serial_number ['${redfish_serial_number}'] 182 183 184Verify Host Off State From PEL 185 [Documentation] Verify Host off state from PEL. 186 [Tags] Verify_Host_Off_State_From_PEL 187 188 Redfish Power Off stack_mode=skip 189 Create Test PEL Log 190 191 ${pel_ids}= Get PEL Log Via BMC CLI 192 ${id}= Get From List ${pel_ids} -1 193 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 194 195 Valid Value pel_host_state ['Off'] 196 197 198Verify BMC Version From PEL 199 [Documentation] Verify BMC Version from PEL. 200 [Tags] Verify_BMC_Version_From_PEL 201 202 Create Test PEL Log 203 204 ${pel_ids}= Get PEL Log Via BMC CLI 205 ${id}= Get From List ${pel_ids} -1 206 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 207 208 ${bmc_version}= Get BMC Version 209 Valid Value bmc_version ['${bmc_version}'] 210 211 212Verify PEL Log After Host Poweron 213 [Documentation] Verify PEL log generation while booting host. 214 [Tags] Verify_PEL_Log_After_Host_Poweron 215 216 Redfish Power Off stack_mode=skip 217 Redfish Purge Event Log 218 Redfish Power On stack_mode=skip 219 220 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 221 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 222 223 # Get BMC created non-infomational error. 224 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 225 226 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 227 228 229Verify BMC Event Log ID 230 [Documentation] Verify BMC Event Log ID from PEL. 231 [Tags] Verify_BMC_Event_Log_ID 232 233 Redfish Purge Event Log 234 Create Test PEL Log 235 236 ${pel_ids}= Get PEL Log Via BMC CLI 237 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 238 239 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 240 # [Private Header]: 241 # [Created at]: 08/24/1928 12:04:06 242 # [Created by]: 0x584D 243 # [Sub-section type]: 0 244 # [Entry Id]: 0x50000BB7 245 # [Platform Log Id]: 0x8200061D 246 # [CSSVER]: 247 # [Section Version]: 1 248 # [Creator Subsystem]: PHYP 249 # [BMC Event Log Id]: 341 <---- BMC event log id value 250 # [Committed at]: 03/25/1920 12:06:22 251 252 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 253 254 # Example of redfish_event_logs output: 255 # redfish_event_logs: 256 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 257 # [Name]: System Event Log Entries 258 # [Members@odata.count]: 1 259 # [@odata.type]: #LogEntryCollection.LogEntryCollection 260 # [Description]: Collection of System Event Log Entries 261 # [Members]: 262 # [0]: 263 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 264 # [Name]: System Event Log Entry 265 # [Severity]: Critical 266 # [EntryType]: Event 267 # [Created]: 2020-04-02T07:25:13+00:00 268 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 269 # [Id]: 235 <----- Event log ID 270 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 271 272 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 273 274 275Verify FRU Callout 276 [Documentation] Verify FRU callout entries from PEL log. 277 [Tags] Verify_FRU_Callout 278 279 Create Test PEL Log FRU Callout 280 281 ${pel_ids}= Get PEL Log Via BMC CLI 282 ${id}= Get From List ${pel_ids} -1 283 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 284 285 # Example of PEL Callout Section from "peltool -i <id>" command. 286 # [Callouts]: 287 # [0]: 288 # [FRU Type]: Normal Hardware FRU 289 # [Priority]: Mandatory, replace all with this type as a unit 290 # [Location Code]: U78DA.ND1.1234567-P0 291 # [Part Number]: F191014 292 # [CCIN]: 2E2D 293 # [Serial Number]: YL2E2D010000 294 # [Callout Count]: 1 295 296 Valid Value pel_callout_section['Callout Count'] ['1'] 297 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 298 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 299 300 # Verify Location Code field of PEL callout with motherboard's Location Code. 301 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 302 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 303 ... ${busctl_output[0].split('"')[1].strip('"')} 304 305 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 306 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 307 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 308 309 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 310 ${busctl_output}= BMC Execute Command 311 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 312 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 313 ... ${busctl_output[0].split('"')[1].strip('"')} 314 315 316Verify Procedure And Symbolic FRU Callout 317 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 318 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 319 320 Create Test PEL Log Procedure And Symbolic FRU Callout 321 322 ${pel_ids}= Get PEL Log Via BMC CLI 323 ${id}= Get From List ${pel_ids} -1 324 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 325 326 # Example of PEL Callout Section from "peltool -i <id>" command. 327 # [Callouts]: 328 # [0]: 329 # [Priority]: Mandatory, replace all with this type as a unit 330 # [Procedure Number]: BMCSP02 331 # [FRU Type]: Maintenance Procedure Required 332 # [1]: 333 # [Priority]: Medium Priority 334 # [Part Number]: SVCDOCS 335 # [FRU Type]: Symbolic FRU 336 # [Callout Count]: 2 337 338 Valid Value pel_callout_section['Callout Count'] ['2'] 339 340 # Verify procedural callout info. 341 342 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 343 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 344 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 345 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 346 347 # Verify procedural callout info. 348 349 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 350 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 351 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 352 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 353 354 355Verify PEL Log Entry For Event Log 356 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 357 [Tags] Verify_PEL_Log_Entry_For_Event_Log 358 359 Redfish Purge Event Log 360 361 # Create an unrecoverable error log. 362 Create Test PEL Log Unrecoverable Error 363 364 ${elog_entry}= Get Event Logs 365 # Example of Redfish event logs: 366 # elog_entry: 367 # [0]: 368 # [Message]: BD802003 event in subsystem: Platform Firmware 369 # [Created]: 2020-04-20T01:55:22+00:00 370 # [Id]: 1 371 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 372 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 373 # [EntryType]: Event 374 # [Severity]: Critical 375 # [Name]: System Event Log Entry 376 377 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 378 379 ${pel_records}= Peltool -l 380 # Example output from 'Peltool -l': 381 # pel_records: 382 # [0x50000023]: 383 # [SRC]: BD802003 384 # [CreatorID]: BMC 385 # [Message]: This is a test error 386 # [CompID]: 0x2000 387 # [PLID]: 0x500053D9 388 # [Commit Time]: 04/20/2020 01:55:22 389 # [Subsystem]: Platform Firmware 390 # [Sev]: Unrecoverable Error 391 392 ${ids}= Get Dictionary Keys ${pel_records} 393 ${id}= Get From List ${ids} 0 394 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 395 ... date_format=%m/%d/%Y %H:%M:%S 396 397 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 398 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 399 Should Contain ${elog_entry[0]['Message']} 400 ... ${pel_records['${id}']['SRC']} ignore_case=True 401 402 Should Be Equal ${redfish_log_time} ${pel_log_time} 403 404 405Verify Delete All PEL 406 [Documentation] Verify deleting all PEL logs. 407 [Tags] Verify_Delete_All_PEL 408 409 Create Test PEL Log 410 Create Test PEL Log 411 Peltool --delete-all False 412 413 ${pel_ids}= Get PEL Log Via BMC CLI 414 Should Be Empty ${pel_ids} 415 416 417Verify Informational Error Log 418 [Documentation] Create an informational error log and verify. 419 [Tags] Verify_Informational_Error_Log 420 421 Redfish Purge Event Log 422 # Create an informational error log. 423 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 424 ${pel_records}= Peltool -lfh 425 426 # An example of information error log data: 427 # { 428 # "0x500006A0": { 429 # "SRC": "BD8D1002", 430 # "Message": "An application had an internal failure", 431 # "PLID": "0x500006A0", 432 # "CreatorID": "BMC", 433 # "Subsystem": "BMC Firmware", 434 # "Commit Time": "10/14/2020 11:41:38", 435 # "Sev": "Informational Event", 436 # "CompID": "0x1000" 437 # } 438 # } 439 440 ${ids}= Get Dictionary Keys ${pel_records} 441 ${id}= Get From List ${ids} 0 442 Should Contain ${pel_records['${id}']['Sev']} Informational 443 444 445Verify Predictable Error Log 446 [Documentation] Create a predictive error and verify. 447 [Tags] Verify_Predictable_Error_Log 448 449 # Create a predictable error log. 450 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 451 ${pel_records}= Peltool -l 452 453 # An example of predictive error log data: 454 # { 455 # "0x5000069E": { 456 # "SRC": "BD8D1002", 457 # "Message": "An application had an internal failure", 458 # "PLID": "0x5000069E", 459 # "CreatorID": "BMC", 460 # "Subsystem": "BMC Firmware", 461 # "Commit Time": "10/14/2020 11:40:07", 462 # "Sev": "Predictive Error", 463 # "CompID": "0x1000" 464 # } 465 # } 466 467 ${pel_ids}= Get PEL Log Via BMC CLI 468 ${id}= Get From List ${pel_ids} -1 469 Should Contain ${pel_records['${id}']['Sev']} Predictive 470 471 472Verify Unrecoverable Error Log 473 [Documentation] Create an unrecoverable error and verify. 474 [Tags] Verify_Unrecoverable_Error_Log 475 476 # Create an internal failure error log. 477 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 478 ${pel_records}= Peltool -l 479 480 # An example of unrecoverable error log data: 481 # { 482 # "0x50000CC5": { 483 # "SRC": "BD8D1002", 484 # "Message": "An application had an internal failure", 485 # "PLID": "0x50000CC5", 486 # "CreatorID": "BMC", 487 # "Subsystem": "BMC Firmware", 488 # "Commit Time": "04/01/2020 16:44:55", 489 # "Sev": "Unrecoverable Error", 490 # "CompID": "0x1000" 491 # } 492 # } 493 494 ${pel_ids}= Get PEL Log Via BMC CLI 495 ${id}= Get From List ${pel_ids} -1 496 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 497 498 499Verify Error Logging Rotation Policy 500 [Documentation] Verify error logging rotation policy. 501 [Tags] Verify_Error_Logging_Rotation_Policy 502 [Template] Error Logging Rotation Policy 503 504 # Error logs to be created % of total logging space when error 505 # log exceeds max limit. 506 Informational BMC 3000 15 507 Predictive BMC 3000 30 508 Unrecoverable BMC 3000 30 509 Informational BMC 1500, Predictive BMC 1500 45 510 Informational BMC 1500, Unrecoverable BMC 1500 45 511 Unrecoverable BMC 1500, Predictive BMC 1500 30 512 513 514Verify Error Logging Rotation Policy With All Types Of Errors 515 [Documentation] Verify error logging rotation policy with all types of errors. 516 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 517 [Template] Error Logging Rotation Policy 518 519 # Error logs to be created % of total logging space when error 520 # log exceeds max limit. 521 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 522 523 524Verify Error Logging Rotation Policy With HOST Error Logs 525 [Documentation] Verify error logging rotation policy for non bmc error logs. 526 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 527 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 528 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 529 ... /tmp/FILE_HOST_INFORMATIONAL 530 [Template] Error Logging Rotation Policy 531 532 # Error logs to be created % of total logging space when error 533 # log exceeds max limit. 534 Informational HOST 3000 15 535 Unrecoverable HOST 3000 30 536 Informational HOST 1500, Informational BMC 1500 30 537 Informational HOST 1500, Unrecoverable BMC 1500 45 538 Unrecoverable HOST 1500, Informational BMC 1500 45 539 Unrecoverable HOST 1500, Predictive BMC 1500 60 540 541 542Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 543 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 544 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 545 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 546 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 547 [Template] Error Logging Rotation Policy 548 549 # Error logs to be created % of total logging space when error 550 # log exceeds max limit. 551 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 552 553 554Verify Old Logs Are Deleted When Count Crosses Max 555 [Documentation] Verify that when the count crosses max, older logs are deleted. 556 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 557 558 Redfish Purge Event Log 559 # Create 3000 error logs. 560 FOR ${count} IN RANGE ${3000} 561 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 562 END 563 564 # Retrieve the IDs of the logs. 565 ${pel_ids}= Get PEL Log Via BMC CLI 566 ${1st_id}= Get From List ${pel_ids} 0 567 ${3000th_id}= Get From List ${pel_ids} 2999 568 569 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 570 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 571 572 # Wait few seconds for error logs rotation to complete. 573 Sleep 10s 574 575 # Now verify that first log is no more available but the 3000th is available. 576 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 577 Should Be True '${status}' == 'FAIL' 578 Should Contain ${output} PEL not found 579 580 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 581 Should Be True '${status}' == 'PASS' 582 Should Not Contain ${output} PEL not found 583 584 585Verify Reverse Order Of PEL Logs 586 [Documentation] Verify PEL command to output PEL logs in reverse order. 587 [Tags] Verify_Reverse_Order_Of_PEL_Logs 588 589 Redfish Purge Event Log 590 591 # Below commands create unrecoverable error log at first and then the predictable error. 592 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 593 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 594 595 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 596 ${pel_records}= peltool -lr 597 598 # It is found that, variables like dictionary always keep items in sorted order that makes 599 # this verification not possible, hence json is used to keep the items original order. 600 ${pel_records}= Convert To String ${pel_records} 601 ${json_string}= Replace String ${pel_records} ' " 602 ${json_object}= Evaluate json.loads('''${json_string}''') json 603 604 ${list}= Convert To List ${json_object} 605 606 ${id1}= Get From List ${list} 0 607 ${id1}= Convert To Integer ${id1} 608 ${id2}= Get From List ${list} 1 609 ${id2}= Convert To Integer ${id2} 610 611 Should Be True ${id1} > ${id2} 612 613 614Verify Total PEL Count 615 [Documentation] Verify total PEL count returned by peltool command. 616 [Tags] Verify_Total_PEL_Count 617 618 # Initially remove all logs. 619 Redfish Purge Event Log 620 621 # Generate a random number between 1-20. 622 ${random}= Evaluate random.randint(1, 20) modules=random 623 624 # Generate predictive error log multiple times. 625 FOR ${count} IN RANGE 0 ${random} 626 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 627 END 628 629 # Check PEL log count via peltool command and compare it with actual generated log count. 630 ${pel_records}= peltool -n 631 632 Should Be Equal ${pel_records['Number of PELs found']} ${random} 633 634 635Verify Listing Information Error 636 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 637 [Tags] Verify_Listing_Information_Error 638 639 # Initially remove all logs. 640 Redfish Purge Event Log 641 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 642 643 # Generate informational logs and verify that it would not get listed by peltool's list command. 644 ${pel_records}= peltool -l 645 ${ids}= Get Dictionary Keys ${pel_records} 646 Should Be Empty ${ids} 647 648 # Verify that information logs get listed using peltool's list command with -lfh option. 649 ${pel_records}= peltool -lfh 650 ${ids}= Get Dictionary Keys ${pel_records} 651 Should Not Be Empty ${ids} 652 ${id}= Get From List ${ids} 0 653 Should Contain ${pel_records['${id}']['Sev']} Informational 654 655 656Verify PEL Delete 657 [Documentation] Verify that peltool command can delete PEL log based on id. 658 [Tags] Verify_PEL_Delete 659 660 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 661 ${pel_ids}= Get PEL Log Via BMC CLI 662 ${id}= Get From List ${pel_ids} -1 663 Peltool -d ${id} False 664 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 665 666 667Verify Mandatory Fields For Predictive Error 668 [Documentation] Verify mandatory fields of predictive errors from pel information. 669 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 670 671 # Inject predictive error. 672 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 673 674 ${pel_ids}= Get PEL Log Via BMC CLI 675 ${pel_id}= Get From List ${pel_ids} -1 676 ${pel_output}= Peltool -i ${pel_id} 677 # Get all fields in predictive error log. 678 ${pel_sections}= Get Dictionary Keys ${pel_output} 679 680 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fileds} 681 682 683Verify Mandatory Fields For Informational Error 684 [Documentation] Verify mandatory fields of informational error from pel information. 685 [Tags] Verify_Mandatory_Fields_For_Informational_Error 686 687 # Inject informational error. 688 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 689 ${pel_records}= Peltool -lfh 690 691 ${ids}= Get Dictionary Keys ${pel_records} 692 ${pel_id}= Get From List ${ids} -1 693 ${pel_output}= Peltool -i ${pel_id} 694 695 # Get all fields in the informational error log. 696 ${pel_sections}= Get Dictionary Keys ${pel_output} 697 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 698 ${contains}= Evaluate "${section}" in "${pel_sections}" 699 Should Be True ${contains} 700 END 701 702 703*** Keywords *** 704 705Error Logging Rotation Policy 706 [Documentation] Verify that when maximum log limit is reached, given error logging type 707 ... are deleted when reached their max allocated space. 708 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 709 710 # Description of argument(s): 711 # error_log Error logs to be created (E.g. Informational BMC 3000 712 # stands for BMC created 3000 informational error logs). 713 # max_allocated_space_percentage The maximum percentage of disk usage for given error 714 # log type when maximum count/log size is reached. 715 # The maximum error log count is 3000. 716 717 Redfish.Login 718 719 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 720 # in BMC during the test. 721 Redfish Purge Event Log 722 723 @{lists}= Split String ${error_log_type} ,${SPACE} 724 725 ${length}= Get Length ${lists} 726 727 FOR ${list} IN RANGE ${length} 728 @{words}= Split String ${lists}[${list}] ${SPACE} 729 Create Error Log ${words}[0] ${words}[1] ${words}[2] 730 END 731 732 # Create an additional error log to exceed max error logs limit. 733 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 734 735 # Delay for BMC to perform delete older error logs when log limit exceeds. 736 Sleep 10s 737 738 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 739 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 740 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 741 742 ${disk_usage_percentage}= Get Disk Usage For Error Logs 743 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 744 ${percent_diff}= Evaluate abs(${percent_diff}) 745 746 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 747 ... Evaluate ${percent_diff} <= 0.5 748 ... ELSE 749 ... Set Variable True 750 751 # Check PEL log count via peltool command and compare it with actual generated log count. 752 ${pel_records}= peltool -n 753 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 754 # Number of logs can be 80% of the total logs created after trimming. 755 ${expected_max_record}= Evaluate 3000 * 0.8 756 757 Run Keyword If ${trimmed_as_expected} == False 758 ... Should Be True ${no_pel_records} <= ${expected_max_record} 759 760 761Create Error Log 762 [Documentation] Create an error log. 763 [Arguments] ${error_severity} ${error_creator} ${count} 764 765 # Description of argument(s): 766 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 767 # error_creator Name of error log's creator(E.g BMC, Host Boot) 768 # count Number of error logs to be generated. 769 770 FOR ${i} IN RANGE 0 ${count} 771 ${cmd}= Set Variable If 772 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 773 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 774 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 775 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 776 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 777 BMC Execute Command ${cmd} 778 END 779 780 781Get Disk Usage For Error Logs 782 [Documentation] Get disk usage percentage for error logs. 783 784 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 785 786 ${usage_output}= Fetch From Left ${usage_output} \/ 787 788 # Convert disk usage unit from KB to MB. 789 ${usage_output}= Evaluate ${usage_output} / 1024 790 791 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 792 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 793 794 [return] ${usage_percent} 795 796 797Get PEL Log IDs 798 [Documentation] Returns the list of PEL log IDs which contains given field's value. 799 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 800 801 # Description of argument(s): 802 # pel_section The section of PEL (e.g. Private Header, User Header). 803 # pel_field The PEL field (e.g. Event Severity, Event Type). 804 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 805 806 ${pel_ids}= Get PEL Log Via BMC CLI 807 @{pel_id_list}= Create List 808 809 FOR ${id} IN @{pel_ids} 810 ${pel_output}= Peltool -i ${id} 811 # Example of PEL output from "peltool -i <id>" command. 812 # [Private Header]: 813 # [Created at]: 08/24/1928 12:04:06 814 # [Created by]: 0x584D 815 # [Sub-section type]: 0 816 # [Entry Id]: 0x50000BB7 817 # [Platform Log Id]: 0x8200061D 818 # [CSSVER]: 819 # [Section Version]: 1 820 # [Creator Subsystem]: PHYP 821 # [BMC Event Log Id]: 341 822 # [Committed at]: 03/25/1920 12:06:22 823 # [User Header]: 824 # [Log Committed by]: 0x4552 825 # [Action Flags]: 826 # [0]: Report Externally 827 # [Subsystem]: I/O Subsystem 828 # [Event Type]: Miscellaneous, Informational Only 829 # [Sub-section type]: 0 830 # [Event Scope]: Entire Platform 831 # [Event Severity]: Informational Event 832 # [Host Transmission]: Not Sent 833 # [Section Version]: 1 834 835 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 836 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 837 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 838 END 839 Sort List ${pel_id_list} 840 841 [Return] ${pel_id_list} 842 843 844Get PEL Log Via BMC CLI 845 [Documentation] Returns the list of PEL IDs using BMC CLI. 846 847 ${pel_records}= Peltool -l 848 ${ids}= Get Dictionary Keys ${pel_records} 849 Sort List ${ids} 850 851 [Return] ${ids} 852 853 854Get PEL Field Value 855 [Documentation] Returns the value of given PEL's field. 856 [Arguments] ${pel_id} ${pel_section} ${pel_field} 857 858 # Description of argument(s): 859 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 860 # pel_section The section of PEL (e.g. Private Header, User Header) 861 # pel_field The PEL field (e.g. Event Severity, Event Type). 862 863 ${pel_output}= Peltool -i ${pel_id} 864 865 # Example of PEL output from "peltool -i <id>" command. 866 # [Private Header]: 867 # [Created at]: 08/24/1928 12:04:06 868 # [Created by]: 0x584D 869 # [Sub-section type]: 0 870 # [Entry Id]: 0x50000BB7 871 # [Platform Log Id]: 0x8200061D 872 # [CSSVER]: 873 # [Section Version]: 1 874 # [Creator Subsystem]: PHYP 875 # [BMC Event Log Id]: 341 876 # [Committed at]: 03/25/1920 12:06:22 877 # [User Header]: 878 # [Log Committed by]: 0x4552 879 # [Action Flags]: 880 # [0]: Report Externally 881 # [Subsystem]: I/O Subsystem 882 # [Event Type]: Miscellaneous, Informational Only 883 # [Sub-section type]: 0 884 # [Event Scope]: Entire Platform 885 # [Event Severity]: Informational Event 886 # [Host Transmission]: Not Sent 887 # [Section Version]: 1 888 889 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 890 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 891 892 [Return] ${pel_field_output} 893