1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 18@{mandatory_Predictive_pel_fileds} Private Header User Header Primary SRC 19... Extended User Header Failing MTMS User Data 20 21*** Test Cases *** 22 23Create Test PEL Log And Verify 24 [Documentation] Create PEL log using busctl command and verify via peltool. 25 [Tags] Create_Test_PEL_Log_And_Verify 26 27 Redfish Purge Event Log 28 Create Test PEL Log 29 ${pel_id}= Get PEL Log Via BMC CLI 30 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 31 32 33Verify PEL Log Details 34 [Documentation] Verify PEL log details via peltool. 35 [Tags] Verify_PEL_Log_Details 36 37 Redfish Purge Event Log 38 39 ${bmc_time1}= CLI Get BMC DateTime 40 Create Test PEL Log 41 ${bmc_time2}= CLI Get BMC DateTime 42 43 ${pel_records}= Peltool -l 44 45 # Example output from 'Peltool -l': 46 # pel_records: 47 # [0x50000012]: 48 # [CreatorID]: BMC 49 # [CompID]: 0x1000 50 # [PLID]: 0x50000012 51 # [Subsystem]: BMC Firmware 52 # [Message]: An application had an internal failure 53 # [SRC]: BD8D1002 54 # [Commit Time]: 03/02/2020 09:35:15 55 # [Sev]: Unrecoverable Error 56 57 ${ids}= Get Dictionary Keys ${pel_records} 58 ${id}= Get From List ${ids} 0 59 60 @{pel_fields}= Create List CreatorID Subsystem Message Sev 61 FOR ${field} IN @{pel_fields} 62 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 63 END 64 65 Valid Value pel_records['${id}']['PLID'] ['${id}'] 66 67 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 68 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 69 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 70 71 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 72 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 73 74 # Convert BMC and PEL time to epoch time before comparing. 75 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 76 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 77 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 78 79 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 80 81 82Verify Mandatory Sections Of Error Log PEL 83 [Documentation] Verify mandatory sections of error log PEL. 84 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 85 86 Create Test PEL Log 87 88 ${pel_ids}= Get PEL Log Via BMC CLI 89 ${pel_id}= Get From List ${pel_ids} -1 90 ${pel_output}= Peltool -i ${pel_id} 91 ${pel_sections}= Get Dictionary Keys ${pel_output} 92 93 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 94 95 96Verify PEL Log Persistence After BMC Reboot 97 [Documentation] Verify PEL log persistence after BMC reboot. 98 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 99 100 Create Test PEL Log 101 ${pel_before_reboot}= Get PEL Log Via BMC CLI 102 103 Redfish OBMC Reboot (off) 104 ${pel_after_reboot}= Get PEL Log Via BMC CLI 105 106 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 107 108 109Verify PEL ID Numbering 110 [Documentation] Verify PEL ID numbering. 111 [Tags] Verify_PEL_ID_Numbering 112 113 Redfish Purge Event Log 114 Create Test PEL Log 115 Create Test PEL Log 116 117 ${pel_ids}= Get PEL Log Via BMC CLI 118 119 # Example of PEL IDs from PEL logs. 120 # [0x50000012]: <--- First PEL ID 121 # [CreatorID]: BMC 122 # [CompID]: 0x1000 123 # [PLID]: 0x50000012 124 # [Subsystem]: BMC Firmware 125 # [Message]: An application had an internal failure 126 # [SRC]: BD8D1002 127 # [Commit Time]: 03/02/2020 09:35:15 128 # [Sev]: Unrecoverable Error 129 # 130 # [0x50000013]: <--- Second PEL ID 131 # [CreatorID]: BMC 132 # [CompID]: 0x1000 133 # [PLID]: 0x50000013 134 # [Subsystem]: BMC Firmware 135 # [Message]: An application had an internal failure 136 # [SRC]: BD8D1002 137 # [Commit Time]: 03/02/2020 09:35:15 138 # [Sev]: Unrecoverable Error 139 140 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 141 142Verify Machine Type Model And Serial Number 143 [Documentation] Verify machine type model and serial number from PEL. 144 [Tags] Verify_Machine_Type_Model_And_Serial_Number 145 146 Create Test PEL Log 147 148 ${pel_ids}= Get PEL Log Via BMC CLI 149 ${id}= Get From List ${pel_ids} -1 150 151 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 152 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 153 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 154 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 155 156 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 157 # [Failing MTMS]: 158 # [Created by]: 0x2000 159 # [Machine Type Model]: 1234-ABC <---- Machine type 160 # [Section Version]: 1 161 # [Serial Number]: ABCDEFG <---- Serial number 162 # [Sub-section type]: 0 163 164 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 165 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 166 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 167 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 168 169 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 170 Valid Value pel_serial_number ['${redfish_serial_number}'] 171 172 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 173 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 174 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 175 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 176 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 177 178 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 179 Valid Value pel_serial_number ['${redfish_serial_number}'] 180 181 182Verify Host Off State From PEL 183 [Documentation] Verify Host off state from PEL. 184 [Tags] Verify_Host_Off_State_From_PEL 185 186 Redfish Power Off stack_mode=skip 187 Create Test PEL Log 188 189 ${pel_ids}= Get PEL Log Via BMC CLI 190 ${id}= Get From List ${pel_ids} -1 191 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 192 193 Valid Value pel_host_state ['Off'] 194 195 196Verify BMC Version From PEL 197 [Documentation] Verify BMC Version from PEL. 198 [Tags] Verify_BMC_Version_From_PEL 199 200 Create Test PEL Log 201 202 ${pel_ids}= Get PEL Log Via BMC CLI 203 ${id}= Get From List ${pel_ids} -1 204 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 205 206 ${bmc_version}= Get BMC Version 207 Valid Value bmc_version ['${bmc_version}'] 208 209 210Verify PEL Log After Host Poweron 211 [Documentation] Verify PEL log generation while booting host. 212 [Tags] Verify_PEL_Log_After_Host_Poweron 213 214 Redfish Power Off stack_mode=skip 215 Redfish Purge Event Log 216 Redfish Power On stack_mode=skip 217 218 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 219 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 220 221 # Get BMC created non-infomational error. 222 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 223 224 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 225 226 227Verify BMC Event Log ID 228 [Documentation] Verify BMC Event Log ID from PEL. 229 [Tags] Verify_BMC_Event_Log_ID 230 231 Redfish Purge Event Log 232 Create Test PEL Log 233 234 ${pel_ids}= Get PEL Log Via BMC CLI 235 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 236 237 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 238 # [Private Header]: 239 # [Created at]: 08/24/1928 12:04:06 240 # [Created by]: 0x584D 241 # [Sub-section type]: 0 242 # [Entry Id]: 0x50000BB7 243 # [Platform Log Id]: 0x8200061D 244 # [CSSVER]: 245 # [Section Version]: 1 246 # [Creator Subsystem]: PHYP 247 # [BMC Event Log Id]: 341 <---- BMC event log id value 248 # [Committed at]: 03/25/1920 12:06:22 249 250 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 251 252 # Example of redfish_event_logs output: 253 # redfish_event_logs: 254 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 255 # [Name]: System Event Log Entries 256 # [Members@odata.count]: 1 257 # [@odata.type]: #LogEntryCollection.LogEntryCollection 258 # [Description]: Collection of System Event Log Entries 259 # [Members]: 260 # [0]: 261 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 262 # [Name]: System Event Log Entry 263 # [Severity]: Critical 264 # [EntryType]: Event 265 # [Created]: 2020-04-02T07:25:13+00:00 266 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 267 # [Id]: 235 <----- Event log ID 268 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 269 270 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 271 272 273Verify FRU Callout 274 [Documentation] Verify FRU callout entries from PEL log. 275 [Tags] Verify_FRU_Callout 276 277 Create Test PEL Log FRU Callout 278 279 ${pel_ids}= Get PEL Log Via BMC CLI 280 ${id}= Get From List ${pel_ids} -1 281 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 282 283 # Example of PEL Callout Section from "peltool -i <id>" command. 284 # [Callouts]: 285 # [0]: 286 # [FRU Type]: Normal Hardware FRU 287 # [Priority]: Mandatory, replace all with this type as a unit 288 # [Location Code]: U78DA.ND1.1234567-P0 289 # [Part Number]: F191014 290 # [CCIN]: 2E2D 291 # [Serial Number]: YL2E2D010000 292 # [Callout Count]: 1 293 294 Valid Value pel_callout_section['Callout Count'] ['1'] 295 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 296 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 297 298 # Verify Location Code field of PEL callout with motherboard's Location Code. 299 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 300 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 301 ... ${busctl_output[0].split('"')[1].strip('"')} 302 303 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 304 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 305 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 306 307 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 308 ${busctl_output}= BMC Execute Command 309 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 310 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 311 ... ${busctl_output[0].split('"')[1].strip('"')} 312 313 314Verify Procedure And Symbolic FRU Callout 315 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 316 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 317 318 Create Test PEL Log Procedure And Symbolic FRU Callout 319 320 ${pel_ids}= Get PEL Log Via BMC CLI 321 ${id}= Get From List ${pel_ids} -1 322 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 323 324 # Example of PEL Callout Section from "peltool -i <id>" command. 325 # [Callouts]: 326 # [0]: 327 # [Priority]: Mandatory, replace all with this type as a unit 328 # [Procedure Number]: BMCSP02 329 # [FRU Type]: Maintenance Procedure Required 330 # [1]: 331 # [Priority]: Medium Priority 332 # [Part Number]: SVCDOCS 333 # [FRU Type]: Symbolic FRU 334 # [Callout Count]: 2 335 336 Valid Value pel_callout_section['Callout Count'] ['2'] 337 338 # Verify procedural callout info. 339 340 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 341 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 342 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 343 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 344 345 # Verify procedural callout info. 346 347 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 348 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 349 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 350 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 351 352 353Verify PEL Log Entry For Event Log 354 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 355 [Tags] Verify_PEL_Log_Entry_For_Event_Log 356 357 Redfish Purge Event Log 358 359 # Create an unrecoverable error log. 360 Create Test PEL Log Unrecoverable Error 361 362 ${elog_entry}= Get Event Logs 363 # Example of Redfish event logs: 364 # elog_entry: 365 # [0]: 366 # [Message]: BD802003 event in subsystem: Platform Firmware 367 # [Created]: 2020-04-20T01:55:22+00:00 368 # [Id]: 1 369 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 370 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 371 # [EntryType]: Event 372 # [Severity]: Critical 373 # [Name]: System Event Log Entry 374 375 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 376 377 ${pel_records}= Peltool -l 378 # Example output from 'Peltool -l': 379 # pel_records: 380 # [0x50000023]: 381 # [SRC]: BD802003 382 # [CreatorID]: BMC 383 # [Message]: This is a test error 384 # [CompID]: 0x2000 385 # [PLID]: 0x500053D9 386 # [Commit Time]: 04/20/2020 01:55:22 387 # [Subsystem]: Platform Firmware 388 # [Sev]: Unrecoverable Error 389 390 ${ids}= Get Dictionary Keys ${pel_records} 391 ${id}= Get From List ${ids} 0 392 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 393 ... date_format=%m/%d/%Y %H:%M:%S 394 395 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 396 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 397 Should Contain ${elog_entry[0]['Message']} 398 ... ${pel_records['${id}']['SRC']} ignore_case=True 399 400 Should Be Equal ${redfish_log_time} ${pel_log_time} 401 402 403Verify Delete All PEL 404 [Documentation] Verify deleting all PEL logs. 405 [Tags] Verify_Delete_All_PEL 406 407 Create Test PEL Log 408 Create Test PEL Log 409 Peltool --delete-all False 410 411 ${pel_ids}= Get PEL Log Via BMC CLI 412 Should Be Empty ${pel_ids} 413 414 415Verify Informational Error Log 416 [Documentation] Create an informational error log and verify. 417 [Tags] Verify_Informational_Error_Log 418 419 Redfish Purge Event Log 420 # Create an informational error log. 421 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 422 ${pel_records}= Peltool -lfh 423 424 # An example of information error log data: 425 # { 426 # "0x500006A0": { 427 # "SRC": "BD8D1002", 428 # "Message": "An application had an internal failure", 429 # "PLID": "0x500006A0", 430 # "CreatorID": "BMC", 431 # "Subsystem": "BMC Firmware", 432 # "Commit Time": "10/14/2020 11:41:38", 433 # "Sev": "Informational Event", 434 # "CompID": "0x1000" 435 # } 436 # } 437 438 ${ids}= Get Dictionary Keys ${pel_records} 439 ${id}= Get From List ${ids} 0 440 Should Contain ${pel_records['${id}']['Sev']} Informational 441 442 443Verify Predictable Error Log 444 [Documentation] Create a predictive error and verify. 445 [Tags] Verify_Predictable_Error_Log 446 447 # Create a predictable error log. 448 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 449 ${pel_records}= Peltool -l 450 451 # An example of predictive error log data: 452 # { 453 # "0x5000069E": { 454 # "SRC": "BD8D1002", 455 # "Message": "An application had an internal failure", 456 # "PLID": "0x5000069E", 457 # "CreatorID": "BMC", 458 # "Subsystem": "BMC Firmware", 459 # "Commit Time": "10/14/2020 11:40:07", 460 # "Sev": "Predictive Error", 461 # "CompID": "0x1000" 462 # } 463 # } 464 465 ${pel_ids}= Get PEL Log Via BMC CLI 466 ${id}= Get From List ${pel_ids} -1 467 Should Contain ${pel_records['${id}']['Sev']} Predictive 468 469 470Verify Unrecoverable Error Log 471 [Documentation] Create an unrecoverable error and verify. 472 [Tags] Verify_Unrecoverable_Error_Log 473 474 # Create an internal failure error log. 475 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 476 ${pel_records}= Peltool -l 477 478 # An example of unrecoverable error log data: 479 # { 480 # "0x50000CC5": { 481 # "SRC": "BD8D1002", 482 # "Message": "An application had an internal failure", 483 # "PLID": "0x50000CC5", 484 # "CreatorID": "BMC", 485 # "Subsystem": "BMC Firmware", 486 # "Commit Time": "04/01/2020 16:44:55", 487 # "Sev": "Unrecoverable Error", 488 # "CompID": "0x1000" 489 # } 490 # } 491 492 ${pel_ids}= Get PEL Log Via BMC CLI 493 ${id}= Get From List ${pel_ids} -1 494 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 495 496 497Verify Error Logging Rotation Policy 498 [Documentation] Verify error logging rotation policy. 499 [Tags] Verify_Error_Logging_Rotation_Policy 500 [Template] Error Logging Rotation Policy 501 502 # Error logs to be created % of total logging space when error 503 # log exceeds max limit. 504 Informational BMC 3000 15 505 Predictive BMC 3000 30 506 Unrecoverable BMC 3000 30 507 Informational BMC 1500, Predictive BMC 1500 45 508 Informational BMC 1500, Unrecoverable BMC 1500 45 509 Unrecoverable BMC 1500, Predictive BMC 1500 30 510 511 512Verify Error Logging Rotation Policy With All Types Of Errors 513 [Documentation] Verify error logging rotation policy with all types of errors. 514 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 515 [Template] Error Logging Rotation Policy 516 517 # Error logs to be created % of total logging space when error 518 # log exceeds max limit. 519 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 520 521 522Verify Error Logging Rotation Policy With HOST Error Logs 523 [Documentation] Verify error logging rotation policy for non bmc error logs. 524 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 525 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 526 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 527 ... /tmp/FILE_HOST_INFORMATIONAL 528 [Template] Error Logging Rotation Policy 529 530 # Error logs to be created % of total logging space when error 531 # log exceeds max limit. 532 Informational HOST 3000 15 533 Unrecoverable HOST 3000 30 534 Informational HOST 1500, Informational BMC 1500 30 535 Informational HOST 1500, Unrecoverable BMC 1500 45 536 Unrecoverable HOST 1500, Informational BMC 1500 45 537 Unrecoverable HOST 1500, Predictive BMC 1500 60 538 539 540Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 541 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 542 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 543 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 544 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 545 [Template] Error Logging Rotation Policy 546 547 # Error logs to be created % of total logging space when error 548 # log exceeds max limit. 549 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 550 551 552Verify Old Logs Are Deleted When Count Crosses Max 553 [Documentation] Verify that when the count crosses max, older logs are deleted. 554 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 555 556 Redfish Purge Event Log 557 # Create 3000 error logs. 558 FOR ${count} IN RANGE ${3000} 559 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 560 END 561 562 # Retrieve the IDs of the logs. 563 ${pel_ids}= Get PEL Log Via BMC CLI 564 ${1st_id}= Get From List ${pel_ids} 0 565 ${3000th_id}= Get From List ${pel_ids} 2999 566 567 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 568 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 569 570 # Wait few seconds for error logs rotation to complete. 571 Sleep 10s 572 573 # Now verify that first log is no more available but the 3000th is available. 574 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 575 Should Be True '${status}' == 'FAIL' 576 Should Contain ${output} PEL not found 577 578 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 579 Should Be True '${status}' == 'PASS' 580 Should Not Contain ${output} PEL not found 581 582 583Verify Reverse Order Of PEL Logs 584 [Documentation] Verify PEL command to output PEL logs in reverse order. 585 [Tags] Verify_Reverse_Order_Of_PEL_Logs 586 587 Redfish Purge Event Log 588 589 # Below commands create unrecoverable error log at first and then the predictable error. 590 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 591 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 592 593 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 594 ${pel_records}= peltool -lr 595 596 # It is found that, variables like dictionary always keep items in sorted order that makes 597 # this verification not possible, hence json is used to keep the items original order. 598 ${pel_records}= Convert To String ${pel_records} 599 ${json_string}= Replace String ${pel_records} ' " 600 ${json_object}= Evaluate json.loads('''${json_string}''') json 601 602 ${list}= Convert To List ${json_object} 603 604 ${id1}= Get From List ${list} 0 605 ${id1}= Convert To Integer ${id1} 606 ${id2}= Get From List ${list} 1 607 ${id2}= Convert To Integer ${id2} 608 609 Should Be True ${id1} > ${id2} 610 611 612Verify Total PEL Count 613 [Documentation] Verify total PEL count returned by peltool command. 614 [Tags] Verify_Total_PEL_Count 615 616 # Initially remove all logs. 617 Redfish Purge Event Log 618 619 # Generate a random number between 1-20. 620 ${random}= Evaluate random.randint(1, 20) modules=random 621 622 # Generate predictive error log multiple times. 623 FOR ${count} IN RANGE 0 ${random} 624 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 625 END 626 627 # Check PEL log count via peltool command and compare it with actual generated log count. 628 ${pel_records}= peltool -n 629 630 Should Be Equal ${pel_records['Number of PELs found']} ${random} 631 632 633Verify Listing Information Error 634 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 635 [Tags] Verify_Listing_Information_Error 636 637 # Initially remove all logs. 638 Redfish Purge Event Log 639 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 640 641 # Generate informational logs and verify that it would not get listed by peltool's list command. 642 ${pel_records}= peltool -l 643 ${ids}= Get Dictionary Keys ${pel_records} 644 Should Be Empty ${ids} 645 646 # Verify that information logs get listed using peltool's list command with -lfh option. 647 ${pel_records}= peltool -lfh 648 ${ids}= Get Dictionary Keys ${pel_records} 649 Should Not Be Empty ${ids} 650 ${id}= Get From List ${ids} 0 651 Should Contain ${pel_records['${id}']['Sev']} Informational 652 653 654Verify PEL Delete 655 [Documentation] Verify that peltool command can delete PEL log based on id. 656 [Tags] Verify_PEL_Delete 657 658 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 659 ${pel_ids}= Get PEL Log Via BMC CLI 660 ${id}= Get From List ${pel_ids} -1 661 Peltool -d ${id} False 662 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 663 664 665Verify Mandatory Fields For Predictive Error 666 [Documentation] Verify mandatory fields of predictive errors from pel information. 667 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 668 669 # Inject predictive error. 670 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 671 672 ${pel_ids}= Get PEL Log Via BMC CLI 673 ${pel_id}= Get From List ${pel_ids} -1 674 ${pel_output}= Peltool -i ${pel_id} 675 # Get all fields in predictive error log. 676 ${pel_sections}= Get Dictionary Keys ${pel_output} 677 678 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fileds} 679 680 681*** Keywords *** 682 683Error Logging Rotation Policy 684 [Documentation] Verify that when maximum log limit is reached, given error logging type 685 ... are deleted when reached their max allocated space. 686 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 687 688 # Description of argument(s): 689 # error_log Error logs to be created (E.g. Informational BMC 3000 690 # stands for BMC created 3000 informational error logs). 691 # max_allocated_space_percentage The maximum percentage of disk usage for given error 692 # log type when maximum count/log size is reached. 693 # The maximum error log count is 3000. 694 695 Redfish.Login 696 697 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 698 # in BMC during the test. 699 Redfish Purge Event Log 700 701 @{lists}= Split String ${error_log_type} ,${SPACE} 702 703 ${length}= Get Length ${lists} 704 705 FOR ${list} IN RANGE ${length} 706 @{words}= Split String ${lists}[${list}] ${SPACE} 707 Create Error Log ${words}[0] ${words}[1] ${words}[2] 708 END 709 710 # Create an additional error log to exceed max error logs limit. 711 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 712 713 # Delay for BMC to perform delete older error logs when log limit exceeds. 714 Sleep 10s 715 716 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 717 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 718 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 719 720 ${disk_usage_percentage}= Get Disk Usage For Error Logs 721 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 722 ${percent_diff}= Evaluate abs(${percent_diff}) 723 724 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 725 ... Evaluate ${percent_diff} <= 0.5 726 ... ELSE 727 ... Set Variable True 728 729 # Check PEL log count via peltool command and compare it with actual generated log count. 730 ${pel_records}= peltool -n 731 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 732 # Number of logs can be 80% of the total logs created after trimming. 733 ${expected_max_record}= Evaluate 3000 * 0.8 734 735 Run Keyword If ${trimmed_as_expected} == False 736 ... Should Be True ${no_pel_records} <= ${expected_max_record} 737 738 739Create Error Log 740 [Documentation] Create an error log. 741 [Arguments] ${error_severity} ${error_creator} ${count} 742 743 # Description of argument(s): 744 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 745 # error_creator Name of error log's creator(E.g BMC, Host Boot) 746 # count Number of error logs to be generated. 747 748 FOR ${i} IN RANGE 0 ${count} 749 ${cmd}= Set Variable If 750 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 751 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 752 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 753 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 754 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 755 BMC Execute Command ${cmd} 756 END 757 758 759Get Disk Usage For Error Logs 760 [Documentation] Get disk usage percentage for error logs. 761 762 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 763 764 ${usage_output}= Fetch From Left ${usage_output} \/ 765 766 # Convert disk usage unit from KB to MB. 767 ${usage_output}= Evaluate ${usage_output} / 1024 768 769 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 770 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 771 772 [return] ${usage_percent} 773 774 775Get PEL Log IDs 776 [Documentation] Returns the list of PEL log IDs which contains given field's value. 777 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 778 779 # Description of argument(s): 780 # pel_section The section of PEL (e.g. Private Header, User Header). 781 # pel_field The PEL field (e.g. Event Severity, Event Type). 782 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 783 784 ${pel_ids}= Get PEL Log Via BMC CLI 785 @{pel_id_list}= Create List 786 787 FOR ${id} IN @{pel_ids} 788 ${pel_output}= Peltool -i ${id} 789 # Example of PEL output from "peltool -i <id>" command. 790 # [Private Header]: 791 # [Created at]: 08/24/1928 12:04:06 792 # [Created by]: 0x584D 793 # [Sub-section type]: 0 794 # [Entry Id]: 0x50000BB7 795 # [Platform Log Id]: 0x8200061D 796 # [CSSVER]: 797 # [Section Version]: 1 798 # [Creator Subsystem]: PHYP 799 # [BMC Event Log Id]: 341 800 # [Committed at]: 03/25/1920 12:06:22 801 # [User Header]: 802 # [Log Committed by]: 0x4552 803 # [Action Flags]: 804 # [0]: Report Externally 805 # [Subsystem]: I/O Subsystem 806 # [Event Type]: Miscellaneous, Informational Only 807 # [Sub-section type]: 0 808 # [Event Scope]: Entire Platform 809 # [Event Severity]: Informational Event 810 # [Host Transmission]: Not Sent 811 # [Section Version]: 1 812 813 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 814 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 815 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 816 END 817 Sort List ${pel_id_list} 818 819 [Return] ${pel_id_list} 820 821 822Get PEL Log Via BMC CLI 823 [Documentation] Returns the list of PEL IDs using BMC CLI. 824 825 ${pel_records}= Peltool -l 826 ${ids}= Get Dictionary Keys ${pel_records} 827 Sort List ${ids} 828 829 [Return] ${ids} 830 831 832Get PEL Field Value 833 [Documentation] Returns the value of given PEL's field. 834 [Arguments] ${pel_id} ${pel_section} ${pel_field} 835 836 # Description of argument(s): 837 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 838 # pel_section The section of PEL (e.g. Private Header, User Header) 839 # pel_field The PEL field (e.g. Event Severity, Event Type). 840 841 ${pel_output}= Peltool -i ${pel_id} 842 843 # Example of PEL output from "peltool -i <id>" command. 844 # [Private Header]: 845 # [Created at]: 08/24/1928 12:04:06 846 # [Created by]: 0x584D 847 # [Sub-section type]: 0 848 # [Entry Id]: 0x50000BB7 849 # [Platform Log Id]: 0x8200061D 850 # [CSSVER]: 851 # [Section Version]: 1 852 # [Creator Subsystem]: PHYP 853 # [BMC Event Log Id]: 341 854 # [Committed at]: 03/25/1920 12:06:22 855 # [User Header]: 856 # [Log Committed by]: 0x4552 857 # [Action Flags]: 858 # [0]: Report Externally 859 # [Subsystem]: I/O Subsystem 860 # [Event Type]: Miscellaneous, Informational Only 861 # [Sub-section type]: 0 862 # [Event Scope]: Entire Platform 863 # [Event Severity]: Informational Event 864 # [Host Transmission]: Not Sent 865 # [Section Version]: 1 866 867 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 868 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 869 870 [Return] ${pel_field_output} 871