1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17@{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 18@{mandatory_Predictive_pel_fileds} Private Header User Header Primary SRC 19... Extended User Header Failing MTMS User Data 20@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 21... Extended User Header User Data 22 23*** Test Cases *** 24 25Create Test PEL Log And Verify 26 [Documentation] Create PEL log using busctl command and verify via peltool. 27 [Tags] Create_Test_PEL_Log_And_Verify 28 29 Redfish Purge Event Log 30 Create Test PEL Log 31 ${pel_id}= Get PEL Log Via BMC CLI 32 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 33 34 35Verify PEL Log Details 36 [Documentation] Verify PEL log details via peltool. 37 [Tags] Verify_PEL_Log_Details 38 39 Redfish Purge Event Log 40 41 ${bmc_time1}= CLI Get BMC DateTime 42 Create Test PEL Log 43 ${bmc_time2}= CLI Get BMC DateTime 44 45 ${pel_records}= Peltool -l 46 47 # Example output from 'Peltool -l': 48 # pel_records: 49 # [0x50000012]: 50 # [CreatorID]: BMC 51 # [CompID]: 0x1000 52 # [PLID]: 0x50000012 53 # [Subsystem]: BMC Firmware 54 # [Message]: An application had an internal failure 55 # [SRC]: BD8D1002 56 # [Commit Time]: 03/02/2020 09:35:15 57 # [Sev]: Unrecoverable Error 58 59 ${ids}= Get Dictionary Keys ${pel_records} 60 ${id}= Get From List ${ids} 0 61 62 @{pel_fields}= Create List CreatorID Subsystem Message Sev 63 FOR ${field} IN @{pel_fields} 64 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 65 END 66 67 Valid Value pel_records['${id}']['PLID'] ['${id}'] 68 69 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 70 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 71 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 72 73 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 74 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 75 76 # Convert BMC and PEL time to epoch time before comparing. 77 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 78 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 79 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 80 81 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 82 83 84Verify Mandatory Sections Of Error Log PEL 85 [Documentation] Verify mandatory sections of error log PEL. 86 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 87 88 Create Test PEL Log 89 90 ${pel_ids}= Get PEL Log Via BMC CLI 91 ${pel_id}= Get From List ${pel_ids} -1 92 ${pel_output}= Peltool -i ${pel_id} 93 ${pel_sections}= Get Dictionary Keys ${pel_output} 94 95 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 96 97 98Verify PEL Log Persistence After BMC Reboot 99 [Documentation] Verify PEL log persistence after BMC reboot. 100 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 101 102 Create Test PEL Log 103 ${pel_before_reboot}= Get PEL Log Via BMC CLI 104 105 Redfish OBMC Reboot (off) 106 ${pel_after_reboot}= Get PEL Log Via BMC CLI 107 108 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 109 110 111Verify PEL ID Numbering 112 [Documentation] Verify PEL ID numbering. 113 [Tags] Verify_PEL_ID_Numbering 114 115 Redfish Purge Event Log 116 Create Test PEL Log 117 Create Test PEL Log 118 119 ${pel_ids}= Get PEL Log Via BMC CLI 120 121 # Example of PEL IDs from PEL logs. 122 # [0x50000012]: <--- First PEL ID 123 # [CreatorID]: BMC 124 # [CompID]: 0x1000 125 # [PLID]: 0x50000012 126 # [Subsystem]: BMC Firmware 127 # [Message]: An application had an internal failure 128 # [SRC]: BD8D1002 129 # [Commit Time]: 03/02/2020 09:35:15 130 # [Sev]: Unrecoverable Error 131 # 132 # [0x50000013]: <--- Second PEL ID 133 # [CreatorID]: BMC 134 # [CompID]: 0x1000 135 # [PLID]: 0x50000013 136 # [Subsystem]: BMC Firmware 137 # [Message]: An application had an internal failure 138 # [SRC]: BD8D1002 139 # [Commit Time]: 03/02/2020 09:35:15 140 # [Sev]: Unrecoverable Error 141 142 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 143 144Verify Machine Type Model And Serial Number 145 [Documentation] Verify machine type model and serial number from PEL. 146 [Tags] Verify_Machine_Type_Model_And_Serial_Number 147 148 Create Test PEL Log 149 150 ${pel_ids}= Get PEL Log Via BMC CLI 151 ${id}= Get From List ${pel_ids} -1 152 153 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 154 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 155 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 156 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 157 158 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 159 # [Failing MTMS]: 160 # [Created by]: 0x2000 161 # [Machine Type Model]: 1234-ABC <---- Machine type 162 # [Section Version]: 1 163 # [Serial Number]: ABCDEFG <---- Serial number 164 # [Sub-section type]: 0 165 166 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 167 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 168 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 169 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 170 171 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 172 Valid Value pel_serial_number ['${redfish_serial_number}'] 173 174 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 175 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 176 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 177 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 178 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 179 180 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 181 Valid Value pel_serial_number ['${redfish_serial_number}'] 182 183 184Verify Host Off State From PEL 185 [Documentation] Verify Host off state from PEL. 186 [Tags] Verify_Host_Off_State_From_PEL 187 188 Redfish Power Off stack_mode=skip 189 Create Test PEL Log 190 191 ${pel_ids}= Get PEL Log Via BMC CLI 192 ${id}= Get From List ${pel_ids} -1 193 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 194 195 Valid Value pel_host_state ['Off'] 196 197 198Verify BMC Version From PEL 199 [Documentation] Verify BMC Version from PEL. 200 [Tags] Verify_BMC_Version_From_PEL 201 202 Create Test PEL Log 203 204 ${pel_ids}= Get PEL Log Via BMC CLI 205 ${id}= Get From List ${pel_ids} -1 206 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 207 208 ${bmc_version}= Get BMC Version 209 Valid Value bmc_version ['${bmc_version}'] 210 211 212Verify PEL Log After Host Poweron 213 [Documentation] Verify PEL log generation while booting host. 214 [Tags] Verify_PEL_Log_After_Host_Poweron 215 216 Redfish Power Off stack_mode=skip 217 Redfish Purge Event Log 218 Redfish Power On stack_mode=skip 219 220 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 221 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 222 223 # Get BMC created non-infomational error. 224 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 225 226 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 227 228 229Verify BMC Event Log ID 230 [Documentation] Verify BMC Event Log ID from PEL. 231 [Tags] Verify_BMC_Event_Log_ID 232 233 Redfish Purge Event Log 234 Create Test PEL Log 235 236 ${pel_ids}= Get PEL Log Via BMC CLI 237 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 238 239 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 240 # [Private Header]: 241 # [Created at]: 08/24/1928 12:04:06 242 # [Created by]: 0x584D 243 # [Sub-section type]: 0 244 # [Entry Id]: 0x50000BB7 245 # [Platform Log Id]: 0x8200061D 246 # [CSSVER]: 247 # [Section Version]: 1 248 # [Creator Subsystem]: PHYP 249 # [BMC Event Log Id]: 341 <---- BMC event log id value 250 # [Committed at]: 03/25/1920 12:06:22 251 252 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 253 254 # Example of redfish_event_logs output: 255 # redfish_event_logs: 256 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 257 # [Name]: System Event Log Entries 258 # [Members@odata.count]: 1 259 # [@odata.type]: #LogEntryCollection.LogEntryCollection 260 # [Description]: Collection of System Event Log Entries 261 # [Members]: 262 # [0]: 263 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 264 # [Name]: System Event Log Entry 265 # [Severity]: Critical 266 # [EntryType]: Event 267 # [Created]: 2020-04-02T07:25:13+00:00 268 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 269 # [Id]: 235 <----- Event log ID 270 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 271 272 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 273 274 275Verify FRU Callout 276 [Documentation] Verify FRU callout entries from PEL log. 277 [Tags] Verify_FRU_Callout 278 279 # Power off the system to avoid unnecessary error logs during test. 280 Redfish Power Off stack_mode=skip 281 Create Test PEL Log FRU Callout 282 283 ${pel_ids}= Get PEL Log Via BMC CLI 284 ${id}= Get From List ${pel_ids} -1 285 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 286 287 # Example of PEL Callout Section from "peltool -i <id>" command. 288 # [Callouts]: 289 # [0]: 290 # [FRU Type]: Normal Hardware FRU 291 # [Priority]: Mandatory, replace all with this type as a unit 292 # [Location Code]: U78DA.ND1.1234567-P0 293 # [Part Number]: F191014 294 # [CCIN]: 2E2D 295 # [Serial Number]: YL2E2D010000 296 # [Callout Count]: 1 297 298 Valid Value pel_callout_section['Callout Count'] ['1'] 299 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 300 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 301 302 # Verify Location Code field of PEL callout with motherboard's Location Code. 303 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 304 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 305 ... ${busctl_output[0].split('"')[1].strip('"')} 306 307 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 308 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 309 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 310 311 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 312 ${busctl_output}= BMC Execute Command 313 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 314 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 315 ... ${busctl_output[0].split('"')[1].strip('"')} 316 317 318Verify Procedure And Symbolic FRU Callout 319 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 320 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 321 322 Create Test PEL Log Procedure And Symbolic FRU Callout 323 324 ${pel_ids}= Get PEL Log Via BMC CLI 325 ${id}= Get From List ${pel_ids} -1 326 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 327 328 # Example of PEL Callout Section from "peltool -i <id>" command. 329 # [Callouts]: 330 # [0]: 331 # [Priority]: Mandatory, replace all with this type as a unit 332 # [Procedure Number]: BMCSP02 333 # [FRU Type]: Maintenance Procedure Required 334 # [1]: 335 # [Priority]: Medium Priority 336 # [Part Number]: SVCDOCS 337 # [FRU Type]: Symbolic FRU 338 # [Callout Count]: 2 339 340 Valid Value pel_callout_section['Callout Count'] ['2'] 341 342 # Verify procedural callout info. 343 344 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 345 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 346 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 347 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 348 349 # Verify procedural callout info. 350 351 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 352 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 353 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 354 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 355 356 357Verify PEL Log Entry For Event Log 358 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 359 [Tags] Verify_PEL_Log_Entry_For_Event_Log 360 361 Redfish Purge Event Log 362 363 # Create an unrecoverable error log. 364 Create Test PEL Log Unrecoverable Error 365 366 ${elog_entry}= Get Event Logs 367 # Example of Redfish event logs: 368 # elog_entry: 369 # [0]: 370 # [Message]: BD802003 event in subsystem: Platform Firmware 371 # [Created]: 2020-04-20T01:55:22+00:00 372 # [Id]: 1 373 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 374 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 375 # [EntryType]: Event 376 # [Severity]: Critical 377 # [Name]: System Event Log Entry 378 379 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 380 381 ${pel_records}= Peltool -l 382 # Example output from 'Peltool -l': 383 # pel_records: 384 # [0x50000023]: 385 # [SRC]: BD802003 386 # [CreatorID]: BMC 387 # [Message]: This is a test error 388 # [CompID]: 0x2000 389 # [PLID]: 0x500053D9 390 # [Commit Time]: 04/20/2020 01:55:22 391 # [Subsystem]: Platform Firmware 392 # [Sev]: Unrecoverable Error 393 394 ${ids}= Get Dictionary Keys ${pel_records} 395 ${id}= Get From List ${ids} 0 396 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 397 ... date_format=%m/%d/%Y %H:%M:%S 398 399 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 400 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 401 Should Contain ${elog_entry[0]['Message']} 402 ... ${pel_records['${id}']['SRC']} ignore_case=True 403 404 Should Be Equal ${redfish_log_time} ${pel_log_time} 405 406 407Verify Delete All PEL 408 [Documentation] Verify deleting all PEL logs. 409 [Tags] Verify_Delete_All_PEL 410 411 Create Test PEL Log 412 Create Test PEL Log 413 Peltool --delete-all False 414 415 ${pel_ids}= Get PEL Log Via BMC CLI 416 Should Be Empty ${pel_ids} 417 418 419Verify Informational Error Log 420 [Documentation] Create an informational error log and verify. 421 [Tags] Verify_Informational_Error_Log 422 423 Redfish Purge Event Log 424 # Create an informational error log. 425 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 426 ${pel_records}= Peltool -lfh 427 428 # An example of information error log data: 429 # { 430 # "0x500006A0": { 431 # "SRC": "BD8D1002", 432 # "Message": "An application had an internal failure", 433 # "PLID": "0x500006A0", 434 # "CreatorID": "BMC", 435 # "Subsystem": "BMC Firmware", 436 # "Commit Time": "10/14/2020 11:41:38", 437 # "Sev": "Informational Event", 438 # "CompID": "0x1000" 439 # } 440 # } 441 442 ${ids}= Get Dictionary Keys ${pel_records} 443 ${id}= Get From List ${ids} 0 444 Should Contain ${pel_records['${id}']['Sev']} Informational 445 446 447Verify Predictable Error Log 448 [Documentation] Create a predictive error and verify. 449 [Tags] Verify_Predictable_Error_Log 450 451 # Create a predictable error log. 452 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 453 ${pel_records}= Peltool -l 454 455 # An example of predictive error log data: 456 # { 457 # "0x5000069E": { 458 # "SRC": "BD8D1002", 459 # "Message": "An application had an internal failure", 460 # "PLID": "0x5000069E", 461 # "CreatorID": "BMC", 462 # "Subsystem": "BMC Firmware", 463 # "Commit Time": "10/14/2020 11:40:07", 464 # "Sev": "Predictive Error", 465 # "CompID": "0x1000" 466 # } 467 # } 468 469 ${pel_ids}= Get PEL Log Via BMC CLI 470 ${id}= Get From List ${pel_ids} -1 471 Should Contain ${pel_records['${id}']['Sev']} Predictive 472 473 474Verify Unrecoverable Error Log 475 [Documentation] Create an unrecoverable error and verify. 476 [Tags] Verify_Unrecoverable_Error_Log 477 478 # Create an internal failure error log. 479 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 480 ${pel_records}= Peltool -l 481 482 # An example of unrecoverable error log data: 483 # { 484 # "0x50000CC5": { 485 # "SRC": "BD8D1002", 486 # "Message": "An application had an internal failure", 487 # "PLID": "0x50000CC5", 488 # "CreatorID": "BMC", 489 # "Subsystem": "BMC Firmware", 490 # "Commit Time": "04/01/2020 16:44:55", 491 # "Sev": "Unrecoverable Error", 492 # "CompID": "0x1000" 493 # } 494 # } 495 496 ${pel_ids}= Get PEL Log Via BMC CLI 497 ${id}= Get From List ${pel_ids} -1 498 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 499 500 501Verify Error Logging Rotation Policy 502 [Documentation] Verify error logging rotation policy. 503 [Tags] Verify_Error_Logging_Rotation_Policy 504 [Template] Error Logging Rotation Policy 505 506 # Error logs to be created % of total logging space when error 507 # log exceeds max limit. 508 Informational BMC 3000 15 509 Predictive BMC 3000 30 510 Unrecoverable BMC 3000 30 511 Informational BMC 1500, Predictive BMC 1500 45 512 Informational BMC 1500, Unrecoverable BMC 1500 45 513 Unrecoverable BMC 1500, Predictive BMC 1500 30 514 515 516Verify Error Logging Rotation Policy With All Types Of Errors 517 [Documentation] Verify error logging rotation policy with all types of errors. 518 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 519 [Template] Error Logging Rotation Policy 520 521 # Error logs to be created % of total logging space when error 522 # log exceeds max limit. 523 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 524 525 526Verify Error Logging Rotation Policy With HOST Error Logs 527 [Documentation] Verify error logging rotation policy for non bmc error logs. 528 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 529 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 530 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 531 ... /tmp/FILE_HOST_INFORMATIONAL 532 [Template] Error Logging Rotation Policy 533 534 # Error logs to be created % of total logging space when error 535 # log exceeds max limit. 536 Informational HOST 3000 15 537 Unrecoverable HOST 3000 30 538 Informational HOST 1500, Informational BMC 1500 30 539 Informational HOST 1500, Unrecoverable BMC 1500 45 540 Unrecoverable HOST 1500, Informational BMC 1500 45 541 Unrecoverable HOST 1500, Predictive BMC 1500 60 542 543 544Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 545 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 546 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 547 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 548 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 549 [Template] Error Logging Rotation Policy 550 551 # Error logs to be created % of total logging space when error 552 # log exceeds max limit. 553 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 554 555 556Verify Old Logs Are Deleted When Count Crosses Max 557 [Documentation] Verify that when the count crosses max, older logs are deleted. 558 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 559 560 Redfish Purge Event Log 561 # Create 3000 error logs. 562 FOR ${count} IN RANGE ${3000} 563 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 564 END 565 566 # Retrieve the IDs of the logs. 567 ${pel_ids}= Get PEL Log Via BMC CLI 568 ${1st_id}= Get From List ${pel_ids} 0 569 ${3000th_id}= Get From List ${pel_ids} 2999 570 571 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 572 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 573 574 # Wait few seconds for error logs rotation to complete. 575 Sleep 10s 576 577 # Now verify that first log is no more available but the 3000th is available. 578 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 579 Should Be True '${status}' == 'FAIL' 580 Should Contain ${output} PEL not found 581 582 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 583 Should Be True '${status}' == 'PASS' 584 Should Not Contain ${output} PEL not found 585 586 587Verify Reverse Order Of PEL Logs 588 [Documentation] Verify PEL command to output PEL logs in reverse order. 589 [Tags] Verify_Reverse_Order_Of_PEL_Logs 590 591 Redfish Purge Event Log 592 593 # Below commands create unrecoverable error log at first and then the predictable error. 594 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 595 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 596 597 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 598 ${pel_records}= peltool -lr 599 600 # It is found that, variables like dictionary always keep items in sorted order that makes 601 # this verification not possible, hence json is used to keep the items original order. 602 ${pel_records}= Convert To String ${pel_records} 603 ${json_string}= Replace String ${pel_records} ' " 604 ${json_object}= Evaluate json.loads('''${json_string}''') json 605 606 ${list}= Convert To List ${json_object} 607 608 ${id1}= Get From List ${list} 0 609 ${id1}= Convert To Integer ${id1} 610 ${id2}= Get From List ${list} 1 611 ${id2}= Convert To Integer ${id2} 612 613 Should Be True ${id1} > ${id2} 614 615 616Verify Total PEL Count 617 [Documentation] Verify total PEL count returned by peltool command. 618 [Tags] Verify_Total_PEL_Count 619 620 # Initially remove all logs. 621 Redfish Purge Event Log 622 623 # Generate a random number between 1-20. 624 ${random}= Evaluate random.randint(1, 20) modules=random 625 626 # Generate predictive error log multiple times. 627 FOR ${count} IN RANGE 0 ${random} 628 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 629 END 630 631 # Check PEL log count via peltool command and compare it with actual generated log count. 632 ${pel_records}= peltool -n 633 634 Should Be Equal ${pel_records['Number of PELs found']} ${random} 635 636 637Verify Listing Information Error 638 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 639 [Tags] Verify_Listing_Information_Error 640 641 # Initially remove all logs. 642 Redfish Purge Event Log 643 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 644 645 # Generate informational logs and verify that it would not get listed by peltool's list command. 646 ${pel_records}= peltool -l 647 ${ids}= Get Dictionary Keys ${pel_records} 648 Should Be Empty ${ids} 649 650 # Verify that information logs get listed using peltool's list command with -lfh option. 651 ${pel_records}= peltool -lfh 652 ${ids}= Get Dictionary Keys ${pel_records} 653 Should Not Be Empty ${ids} 654 ${id}= Get From List ${ids} 0 655 Should Contain ${pel_records['${id}']['Sev']} Informational 656 657 658Verify PEL Delete 659 [Documentation] Verify that peltool command can delete PEL log based on id. 660 [Tags] Verify_PEL_Delete 661 662 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 663 ${pel_ids}= Get PEL Log Via BMC CLI 664 ${id}= Get From List ${pel_ids} -1 665 Peltool -d ${id} False 666 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 667 668 669Verify Mandatory Fields For Predictive Error 670 [Documentation] Verify mandatory fields of predictive errors from pel information. 671 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 672 673 # Inject predictive error. 674 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 675 676 ${pel_ids}= Get PEL Log Via BMC CLI 677 ${pel_id}= Get From List ${pel_ids} -1 678 ${pel_output}= Peltool -i ${pel_id} 679 # Get all fields in predictive error log. 680 ${pel_sections}= Get Dictionary Keys ${pel_output} 681 682 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fileds} 683 684 685Verify Mandatory Fields For Informational Error 686 [Documentation] Verify mandatory fields of informational error from pel information. 687 [Tags] Verify_Mandatory_Fields_For_Informational_Error 688 689 # Inject informational error. 690 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 691 ${pel_records}= Peltool -lfh 692 693 ${ids}= Get Dictionary Keys ${pel_records} 694 ${pel_id}= Get From List ${ids} -1 695 ${pel_output}= Peltool -i ${pel_id} 696 697 # Get all fields in the informational error log. 698 ${pel_sections}= Get Dictionary Keys ${pel_output} 699 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 700 ${contains}= Evaluate "${section}" in "${pel_sections}" 701 Should Be True ${contains} 702 END 703 704 705*** Keywords *** 706 707Error Logging Rotation Policy 708 [Documentation] Verify that when maximum log limit is reached, given error logging type 709 ... are deleted when reached their max allocated space. 710 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 711 712 # Description of argument(s): 713 # error_log Error logs to be created (E.g. Informational BMC 3000 714 # stands for BMC created 3000 informational error logs). 715 # max_allocated_space_percentage The maximum percentage of disk usage for given error 716 # log type when maximum count/log size is reached. 717 # The maximum error log count is 3000. 718 719 Redfish.Login 720 721 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 722 # in BMC during the test. 723 Redfish Purge Event Log 724 725 @{lists}= Split String ${error_log_type} ,${SPACE} 726 727 ${length}= Get Length ${lists} 728 729 FOR ${list} IN RANGE ${length} 730 @{words}= Split String ${lists}[${list}] ${SPACE} 731 Create Error Log ${words}[0] ${words}[1] ${words}[2] 732 END 733 734 # Create an additional error log to exceed max error logs limit. 735 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 736 737 # Delay for BMC to perform delete older error logs when log limit exceeds. 738 Sleep 10s 739 740 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 741 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 742 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 743 744 ${disk_usage_percentage}= Get Disk Usage For Error Logs 745 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 746 ${percent_diff}= Evaluate abs(${percent_diff}) 747 748 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 749 ... Evaluate ${percent_diff} <= 0.5 750 ... ELSE 751 ... Set Variable True 752 753 # Check PEL log count via peltool command and compare it with actual generated log count. 754 ${pel_records}= peltool -n 755 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 756 # Number of logs can be 80% of the total logs created after trimming. 757 ${expected_max_record}= Evaluate 3000 * 0.8 758 759 Run Keyword If ${trimmed_as_expected} == False 760 ... Should Be True ${no_pel_records} <= ${expected_max_record} 761 762 763Create Error Log 764 [Documentation] Create an error log. 765 [Arguments] ${error_severity} ${error_creator} ${count} 766 767 # Description of argument(s): 768 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 769 # error_creator Name of error log's creator(E.g BMC, Host Boot) 770 # count Number of error logs to be generated. 771 772 FOR ${i} IN RANGE 0 ${count} 773 ${cmd}= Set Variable If 774 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 775 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 776 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 777 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 778 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 779 BMC Execute Command ${cmd} 780 END 781 782 783Get Disk Usage For Error Logs 784 [Documentation] Get disk usage percentage for error logs. 785 786 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 787 788 ${usage_output}= Fetch From Left ${usage_output} \/ 789 790 # Convert disk usage unit from KB to MB. 791 ${usage_output}= Evaluate ${usage_output} / 1024 792 793 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 794 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 795 796 [return] ${usage_percent} 797 798 799Get PEL Log IDs 800 [Documentation] Returns the list of PEL log IDs which contains given field's value. 801 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 802 803 # Description of argument(s): 804 # pel_section The section of PEL (e.g. Private Header, User Header). 805 # pel_field The PEL field (e.g. Event Severity, Event Type). 806 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 807 808 ${pel_ids}= Get PEL Log Via BMC CLI 809 @{pel_id_list}= Create List 810 811 FOR ${id} IN @{pel_ids} 812 ${pel_output}= Peltool -i ${id} 813 # Example of PEL output from "peltool -i <id>" command. 814 # [Private Header]: 815 # [Created at]: 08/24/1928 12:04:06 816 # [Created by]: 0x584D 817 # [Sub-section type]: 0 818 # [Entry Id]: 0x50000BB7 819 # [Platform Log Id]: 0x8200061D 820 # [CSSVER]: 821 # [Section Version]: 1 822 # [Creator Subsystem]: PHYP 823 # [BMC Event Log Id]: 341 824 # [Committed at]: 03/25/1920 12:06:22 825 # [User Header]: 826 # [Log Committed by]: 0x4552 827 # [Action Flags]: 828 # [0]: Report Externally 829 # [Subsystem]: I/O Subsystem 830 # [Event Type]: Miscellaneous, Informational Only 831 # [Sub-section type]: 0 832 # [Event Scope]: Entire Platform 833 # [Event Severity]: Informational Event 834 # [Host Transmission]: Not Sent 835 # [Section Version]: 1 836 837 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 838 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 839 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 840 END 841 Sort List ${pel_id_list} 842 843 [Return] ${pel_id_list} 844 845 846Get PEL Log Via BMC CLI 847 [Documentation] Returns the list of PEL IDs using BMC CLI. 848 849 ${pel_records}= Peltool -l 850 ${ids}= Get Dictionary Keys ${pel_records} 851 Sort List ${ids} 852 853 [Return] ${ids} 854 855 856Get PEL Field Value 857 [Documentation] Returns the value of given PEL's field. 858 [Arguments] ${pel_id} ${pel_section} ${pel_field} 859 860 # Description of argument(s): 861 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 862 # pel_section The section of PEL (e.g. Private Header, User Header) 863 # pel_field The PEL field (e.g. Event Severity, Event Type). 864 865 ${pel_output}= Peltool -i ${pel_id} 866 867 # Example of PEL output from "peltool -i <id>" command. 868 # [Private Header]: 869 # [Created at]: 08/24/1928 12:04:06 870 # [Created by]: 0x584D 871 # [Sub-section type]: 0 872 # [Entry Id]: 0x50000BB7 873 # [Platform Log Id]: 0x8200061D 874 # [CSSVER]: 875 # [Section Version]: 1 876 # [Creator Subsystem]: PHYP 877 # [BMC Event Log Id]: 341 878 # [Committed at]: 03/25/1920 12:06:22 879 # [User Header]: 880 # [Log Committed by]: 0x4552 881 # [Action Flags]: 882 # [0]: Report Externally 883 # [Subsystem]: I/O Subsystem 884 # [Event Type]: Miscellaneous, Informational Only 885 # [Sub-section type]: 0 886 # [Event Scope]: Entire Platform 887 # [Event Severity]: Informational Event 888 # [Host Transmission]: Not Sent 889 # [Section Version]: 1 890 891 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 892 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 893 894 [Return] ${pel_field_output} 895