1 *** Settings *** 2 Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4 Library ../../lib/pel_utils.py 5 Variables ../../data/pel_variables.py 6 Resource ../../lib/list_utils.robot 7 Resource ../../lib/logging_utils.robot 8 Resource ../../lib/connection_client.robot 9 Resource ../../lib/openbmc_ffdc.robot 10 11 Test Setup Redfish.Login 12 Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15 *** Variables *** 16 17 @{mandatory_pel_fileds} Private Header User Header Primary SRC Extended User Header Failing MTMS 18 19 20 *** Test Cases *** 21 22 Create Test PEL Log And Verify 23 [Documentation] Create PEL log using busctl command and verify via peltool. 24 [Tags] Create_Test_PEL_Log_And_Verify 25 26 Redfish Purge Event Log 27 Create Test PEL Log 28 ${pel_id}= Get PEL Log Via BMC CLI 29 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 30 31 32 Verify PEL Log Details 33 [Documentation] Verify PEL log details via peltool. 34 [Tags] Verify_PEL_Log_Details 35 36 Redfish Purge Event Log 37 38 ${bmc_time1}= CLI Get BMC DateTime 39 Create Test PEL Log 40 ${bmc_time2}= CLI Get BMC DateTime 41 42 ${pel_records}= Peltool -l 43 44 # Example output from 'Peltool -l': 45 # pel_records: 46 # [0x50000012]: 47 # [CreatorID]: BMC 48 # [CompID]: 0x1000 49 # [PLID]: 0x50000012 50 # [Subsystem]: BMC Firmware 51 # [Message]: An application had an internal failure 52 # [SRC]: BD8D1002 53 # [Commit Time]: 03/02/2020 09:35:15 54 # [Sev]: Unrecoverable Error 55 56 ${ids}= Get Dictionary Keys ${pel_records} 57 ${id}= Get From List ${ids} 0 58 59 @{pel_fields}= Create List CreatorID Subsystem Message Sev 60 FOR ${field} IN @{pel_fields} 61 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 62 END 63 64 Valid Value pel_records['${id}']['PLID'] ['${id}'] 65 66 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 67 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 68 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 69 70 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 71 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 72 73 # Convert BMC and PEL time to epoch time before comparing. 74 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 75 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 76 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 77 78 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 79 80 81 Verify Mandatory Sections Of Error Log PEL 82 [Documentation] Verify mandatory sections of error log PEL. 83 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 84 85 Create Test PEL Log 86 87 ${pel_ids}= Get PEL Log Via BMC CLI 88 ${pel_id}= Get From List ${pel_ids} -1 89 ${pel_output}= Peltool -i ${pel_id} 90 ${pel_sections}= Get Dictionary Keys ${pel_output} 91 92 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fileds} 93 94 95 Verify PEL Log Persistence After BMC Reboot 96 [Documentation] Verify PEL log persistence after BMC reboot. 97 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 98 99 Create Test PEL Log 100 ${pel_before_reboot}= Get PEL Log Via BMC CLI 101 102 Redfish OBMC Reboot (off) 103 ${pel_after_reboot}= Get PEL Log Via BMC CLI 104 105 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 106 107 108 Verify PEL ID Numbering 109 [Documentation] Verify PEL ID numbering. 110 [Tags] Verify_PEL_ID_Numbering 111 112 Redfish Purge Event Log 113 Create Test PEL Log 114 Create Test PEL Log 115 116 ${pel_ids}= Get PEL Log Via BMC CLI 117 118 # Example of PEL IDs from PEL logs. 119 # [0x50000012]: <--- First PEL ID 120 # [CreatorID]: BMC 121 # [CompID]: 0x1000 122 # [PLID]: 0x50000012 123 # [Subsystem]: BMC Firmware 124 # [Message]: An application had an internal failure 125 # [SRC]: BD8D1002 126 # [Commit Time]: 03/02/2020 09:35:15 127 # [Sev]: Unrecoverable Error 128 # 129 # [0x50000013]: <--- Second PEL ID 130 # [CreatorID]: BMC 131 # [CompID]: 0x1000 132 # [PLID]: 0x50000013 133 # [Subsystem]: BMC Firmware 134 # [Message]: An application had an internal failure 135 # [SRC]: BD8D1002 136 # [Commit Time]: 03/02/2020 09:35:15 137 # [Sev]: Unrecoverable Error 138 139 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 140 141 Verify Machine Type Model And Serial Number 142 [Documentation] Verify machine type model and serial number from PEL. 143 [Tags] Verify_Machine_Type_Model_And_Serial_Number 144 145 Create Test PEL Log 146 147 ${pel_ids}= Get PEL Log Via BMC CLI 148 ${id}= Get From List ${pel_ids} -1 149 150 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 151 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 152 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 153 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 154 155 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 156 # [Failing MTMS]: 157 # [Created by]: 0x2000 158 # [Machine Type Model]: 1234-ABC <---- Machine type 159 # [Section Version]: 1 160 # [Serial Number]: ABCDEFG <---- Serial number 161 # [Sub-section type]: 0 162 163 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 164 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 165 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 166 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 167 168 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 169 Valid Value pel_serial_number ['${redfish_serial_number}'] 170 171 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 172 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 173 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 174 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 175 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 176 177 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 178 Valid Value pel_serial_number ['${redfish_serial_number}'] 179 180 181 Verify Host Off State From PEL 182 [Documentation] Verify Host off state from PEL. 183 [Tags] Verify_Host_Off_State_From_PEL 184 185 Redfish Power Off stack_mode=skip 186 Create Test PEL Log 187 188 ${pel_ids}= Get PEL Log Via BMC CLI 189 ${id}= Get From List ${pel_ids} -1 190 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 191 192 Valid Value pel_host_state ['Off'] 193 194 195 Verify BMC Version From PEL 196 [Documentation] Verify BMC Version from PEL. 197 [Tags] Verify_BMC_Version_From_PEL 198 199 Create Test PEL Log 200 201 ${pel_ids}= Get PEL Log Via BMC CLI 202 ${id}= Get From List ${pel_ids} -1 203 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 204 205 ${bmc_version}= Get BMC Version 206 Valid Value bmc_version ['${bmc_version}'] 207 208 209 Verify PEL Log After Host Poweron 210 [Documentation] Verify PEL log generation while booting host. 211 [Tags] Verify_PEL_Log_After_Host_Poweron 212 213 Redfish Power Off stack_mode=skip 214 Redfish Purge Event Log 215 Redfish Power On stack_mode=skip 216 217 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 218 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 219 220 # Get BMC created non-infomational error. 221 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 222 223 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 224 225 226 Verify BMC Event Log ID 227 [Documentation] Verify BMC Event Log ID from PEL. 228 [Tags] Verify_BMC_Event_Log_ID 229 230 Redfish Purge Event Log 231 Create Test PEL Log 232 233 ${pel_ids}= Get PEL Log Via BMC CLI 234 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 235 236 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 237 # [Private Header]: 238 # [Created at]: 08/24/1928 12:04:06 239 # [Created by]: 0x584D 240 # [Sub-section type]: 0 241 # [Entry Id]: 0x50000BB7 242 # [Platform Log Id]: 0x8200061D 243 # [CSSVER]: 244 # [Section Version]: 1 245 # [Creator Subsystem]: PHYP 246 # [BMC Event Log Id]: 341 <---- BMC event log id value 247 # [Committed at]: 03/25/1920 12:06:22 248 249 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 250 251 # Example of redfish_event_logs output: 252 # redfish_event_logs: 253 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 254 # [Name]: System Event Log Entries 255 # [Members@odata.count]: 1 256 # [@odata.type]: #LogEntryCollection.LogEntryCollection 257 # [Description]: Collection of System Event Log Entries 258 # [Members]: 259 # [0]: 260 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 261 # [Name]: System Event Log Entry 262 # [Severity]: Critical 263 # [EntryType]: Event 264 # [Created]: 2020-04-02T07:25:13+00:00 265 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 266 # [Id]: 235 <----- Event log ID 267 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 268 269 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 270 271 272 Verify FRU Callout 273 [Documentation] Verify FRU callout entries from PEL log. 274 [Tags] Verify_FRU_Callout 275 276 Create Test PEL Log FRU Callout 277 278 ${pel_ids}= Get PEL Log Via BMC CLI 279 ${id}= Get From List ${pel_ids} -1 280 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 281 282 # Example of PEL Callout Section from "peltool -i <id>" command. 283 # [Callouts]: 284 # [0]: 285 # [FRU Type]: Normal Hardware FRU 286 # [Priority]: Mandatory, replace all with this type as a unit 287 # [Location Code]: U78DA.ND1.1234567-P0 288 # [Part Number]: F191014 289 # [CCIN]: 2E2D 290 # [Serial Number]: YL2E2D010000 291 # [Callout Count]: 1 292 293 Valid Value pel_callout_section['Callout Count'] ['1'] 294 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 295 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 296 297 # Verify Location Code field of PEL callout with motherboard's Location Code. 298 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 299 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 300 ... ${busctl_output[0].split('"')[1].strip('"')} 301 302 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 303 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 304 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 305 306 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 307 ${busctl_output}= BMC Execute Command 308 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 309 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 310 ... ${busctl_output[0].split('"')[1].strip('"')} 311 312 313 Verify Procedure And Symbolic FRU Callout 314 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 315 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 316 317 Create Test PEL Log Procedure And Symbolic FRU Callout 318 319 ${pel_ids}= Get PEL Log Via BMC CLI 320 ${id}= Get From List ${pel_ids} -1 321 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 322 323 # Example of PEL Callout Section from "peltool -i <id>" command. 324 # [Callouts]: 325 # [0]: 326 # [Priority]: Mandatory, replace all with this type as a unit 327 # [Procedure Number]: BMCSP02 328 # [FRU Type]: Maintenance Procedure Required 329 # [1]: 330 # [Priority]: Medium Priority 331 # [Part Number]: SVCDOCS 332 # [FRU Type]: Symbolic FRU 333 # [Callout Count]: 2 334 335 Valid Value pel_callout_section['Callout Count'] ['2'] 336 337 # Verify procedural callout info. 338 339 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 340 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 341 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 342 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 343 344 # Verify procedural callout info. 345 346 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 347 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 348 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 349 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 350 351 352 Verify PEL Log Entry For Event Log 353 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 354 [Tags] Verify_PEL_Log_Entry_For_Event_Log 355 356 Redfish Purge Event Log 357 # Create an internal failure error log. 358 BMC Execute Command ${CMD_INTERNAL_FAILURE} 359 360 ${elog_entry}= Get Event Logs 361 # Example of Redfish event logs: 362 # elog_entry: 363 # [0]: 364 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 365 # [Created]: 2020-04-20T01:55:22+00:00 366 # [Id]: 1 367 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 368 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 369 # [EntryType]: Event 370 # [Severity]: Critical 371 # [Name]: System Event Log Entry 372 373 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 374 375 ${pel_records}= Peltool -l 376 # Example output from 'Peltool -l': 377 # pel_records: 378 # [0x50000023]: 379 # [SRC]: BD8D1002 380 # [CreatorID]: BMC 381 # [Message]: An application had an internal failure 382 # [CompID]: 0x1000 383 # [PLID]: 0x50000023 384 # [Commit Time]: 04/20/2020 01:55:22 385 # [Subsystem]: BMC Firmware 386 # [Sev]: Unrecoverable Error 387 388 ${ids}= Get Dictionary Keys ${pel_records} 389 ${id}= Get From List ${ids} 0 390 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 391 ... date_format=%m/%d/%Y %H:%M:%S 392 393 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 394 Should Contain Any ${pel_records['${id}']['Message']} internal failure ignore_case=True 395 Should Contain ${elog_entry[0]['Message']} 396 ... ${pel_records['${id}']['SRC']} event in subsystem: BMC Firmware ignore_case=True 397 398 Should Be Equal ${redfish_log_time} ${pel_log_time} 399 400 401 Verify Delete All PEL 402 [Documentation] Verify deleting all PEL logs. 403 [Tags] Verify_Delete_All_PEL 404 405 Create Test PEL Log 406 Create Test PEL Log 407 Peltool --delete-all False 408 409 ${pel_ids}= Get PEL Log Via BMC CLI 410 Should Be Empty ${pel_ids} 411 412 413 Verify Informational Error Log 414 [Documentation] Create an informational error log and verify. 415 [Tags] Verify_Informational_Error_Log 416 417 Redfish Purge Event Log 418 # Create an informational error log. 419 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 420 ${pel_records}= Peltool -lfh 421 422 # An example of information error log data: 423 # { 424 # "0x500006A0": { 425 # "SRC": "BD8D1002", 426 # "Message": "An application had an internal failure", 427 # "PLID": "0x500006A0", 428 # "CreatorID": "BMC", 429 # "Subsystem": "BMC Firmware", 430 # "Commit Time": "10/14/2020 11:41:38", 431 # "Sev": "Informational Event", 432 # "CompID": "0x1000" 433 # } 434 # } 435 436 ${ids}= Get Dictionary Keys ${pel_records} 437 ${id}= Get From List ${ids} 0 438 Should Contain ${pel_records['${id}']['Sev']} Informational 439 440 441 Verify Predictable Error Log 442 [Documentation] Create a predictive error and verify. 443 [Tags] Verify_Predictable_Error_Log 444 445 # Create a predictable error log. 446 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 447 ${pel_records}= Peltool -l 448 449 # An example of predictive error log data: 450 # { 451 # "0x5000069E": { 452 # "SRC": "BD8D1002", 453 # "Message": "An application had an internal failure", 454 # "PLID": "0x5000069E", 455 # "CreatorID": "BMC", 456 # "Subsystem": "BMC Firmware", 457 # "Commit Time": "10/14/2020 11:40:07", 458 # "Sev": "Predictive Error", 459 # "CompID": "0x1000" 460 # } 461 # } 462 463 ${pel_ids}= Get PEL Log Via BMC CLI 464 ${id}= Get From List ${pel_ids} -1 465 Should Contain ${pel_records['${id}']['Sev']} Predictive 466 467 468 Verify Unrecoverable Error Log 469 [Documentation] Create an unrecoverable error and verify. 470 [Tags] Verify_Unrecoverable_Error_Log 471 472 # Create an internal failure error log. 473 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 474 ${pel_records}= Peltool -l 475 476 # An example of unrecoverable error log data: 477 # { 478 # "0x50000CC5": { 479 # "SRC": "BD8D1002", 480 # "Message": "An application had an internal failure", 481 # "PLID": "0x50000CC5", 482 # "CreatorID": "BMC", 483 # "Subsystem": "BMC Firmware", 484 # "Commit Time": "04/01/2020 16:44:55", 485 # "Sev": "Unrecoverable Error", 486 # "CompID": "0x1000" 487 # } 488 # } 489 490 ${pel_ids}= Get PEL Log Via BMC CLI 491 ${id}= Get From List ${pel_ids} -1 492 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 493 494 495 Verify Error Logging Rotation Policy 496 [Documentation] Verify error logging rotation policy. 497 [Tags] Verify_Error_Logging_Rotation_Policy 498 [Template] Error Logging Rotation Policy 499 500 # Error logs to be created % of total logging space when error 501 # log exceeds max limit. 502 Informational BMC 3000 15 503 Predictive BMC 3000 30 504 Unrecoverable BMC 3000 30 505 Informational BMC 1500, Predictive BMC 1500 45 506 Informational BMC 1500, Unrecoverable BMC 1500 45 507 Unrecoverable BMC 1500, Predictive BMC 1500 30 508 509 510 Verify Error Logging Rotation Policy With All Types Of Errors 511 [Documentation] Verify error logging rotation policy with all types of errors. 512 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Errors 513 [Template] Error Logging Rotation Policy 514 515 # Error logs to be created % of total logging space when error 516 # log exceeds max limit. 517 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 518 519 520 Verify Error Logging Rotation Policy With HOST Error Logs 521 [Documentation] Verify error logging rotation policy for non bmc error logs. 522 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 523 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 524 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 525 ... /tmp/FILE_HOST_INFORMATIONAL 526 [Template] Error Logging Rotation Policy 527 528 # Error logs to be created % of total logging space when error 529 # log exceeds max limit. 530 Informational HOST 3000 15 531 Unrecoverable HOST 3000 30 532 Informational HOST 1500, Informational BMC 1500 30 533 Informational HOST 1500, Unrecoverable BMC 1500 45 534 Unrecoverable HOST 1500, Informational BMC 1500 45 535 Unrecoverable HOST 1500, Predictive BMC 1500 60 536 537 538 Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 539 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 540 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 541 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 542 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 543 [Template] Error Logging Rotation Policy 544 545 # Error logs to be created % of total logging space when error 546 # log exceeds max limit. 547 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 548 549 550 Verify Old Logs Are Deleted When Count Crosses Max 551 [Documentation] Verify that when the count crosses max, older logs are deleted. 552 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 553 554 Redfish Purge Event Log 555 # Create 3000 error logs. 556 FOR ${count} IN RANGE ${3000} 557 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 558 END 559 560 # Retrieve the IDs of the logs. 561 ${pel_ids}= Get PEL Log Via BMC CLI 562 ${1st_id}= Get From List ${pel_ids} 0 563 ${3000th_id}= Get From List ${pel_ids} 2999 564 565 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 566 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 567 568 # Wait few seconds for error logs rotation to complete. 569 Sleep 10s 570 571 # Now verify that first log is no more available but the 3000th is available. 572 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 573 Should Be True '${status}' == 'FAIL' 574 Should Contain ${output} PEL not found 575 576 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 577 Should Be True '${status}' == 'PASS' 578 Should Not Contain ${output} PEL not found 579 580 581 Verify Reverse Order Of PEL Logs 582 [Documentation] Verify PEL command to output PEL logs in reverse order. 583 [Tags] Verify_Reverse_PEL_Logs 584 585 Redfish Purge Event Log 586 587 # Below commands create unrecoverable error log at first and then the predictable error. 588 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 589 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 590 591 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 592 ${pel_records}= peltool -lr 593 594 # It is found that, variables like dictionary always keep items in sorted order that makes 595 # this verification not possible, hence json is used to keep the items original order. 596 ${pel_records}= Convert To String ${pel_records} 597 ${json_string}= Replace String ${pel_records} ' " 598 ${json_object}= Evaluate json.loads('''${json_string}''') json 599 600 ${list}= Convert To List ${json_object} 601 602 ${id1}= Get From List ${list} 0 603 ${id1}= Convert To Integer ${id1} 604 ${id2}= Get From List ${list} 1 605 ${id2}= Convert To Integer ${id2} 606 607 Should Be True ${id1} > ${id2} 608 609 610 Verify Total PEL Count 611 [Documentation] Verify total PEL count returned by peltool command. 612 [Tags] Verify_Total_PEL_Count 613 614 # Initially remove all logs. 615 Redfish Purge Event Log 616 617 # Generate a random number between 1-20. 618 ${random}= Evaluate random.randint(1, 20) modules=random 619 620 # Generate predictive error log multiple times. 621 FOR ${count} IN RANGE 0 ${random} 622 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 623 END 624 625 # Check PEL log count via peltool command and compare it with actual generated log count. 626 ${pel_records}= peltool -n 627 628 Should Be Equal ${pel_records['Number of PELs found']} ${random} 629 630 631 Verify Listing Information Error 632 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 633 [Tags] Verify_Listing_Information_Error 634 635 # Initially remove all logs. 636 Redfish Purge Event Log 637 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 638 639 # Generate informational logs and verify that it would not get listed by peltool's list command. 640 ${pel_records}= peltool -l 641 ${ids}= Get Dictionary Keys ${pel_records} 642 Should Be Empty ${ids} 643 644 # Verify that information logs get listed using peltool's list command with -lfh option. 645 ${pel_records}= peltool -lfh 646 ${ids}= Get Dictionary Keys ${pel_records} 647 Should Not Be Empty ${ids} 648 ${id}= Get From List ${ids} 0 649 Should Contain ${pel_records['${id}']['Sev']} Informational 650 651 652 Verify PEL Delete 653 [Documentation] Verify that peltool command can delete PEL log based on id. 654 [Tags] Verify_PEL_Delete 655 656 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 657 ${pel_ids}= Get PEL Log Via BMC CLI 658 ${id}= Get From List ${pel_ids} -1 659 Peltool -d ${id} False 660 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 661 662 663 *** Keywords *** 664 665 Error Logging Rotation Policy 666 [Documentation] Verify that when maximum log limit is reached, given error logging type 667 ... are deleted when reached their max allocated space. 668 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 669 670 # Description of argument(s): 671 # error_log Error logs to be created (E.g. Informational BMC 3000 672 # stands for BMC created 3000 informational error logs). 673 # max_allocated_space_percentage The maximum percentage of disk usage for given error 674 # log type when maximum count/log size is reached. 675 # The maximum error log count is 3000. 676 677 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 678 # in BMC during the test. 679 Redfish Purge Event Log 680 681 @{lists}= Split String ${error_log_type} ,${SPACE} 682 683 ${length}= Get Length ${lists} 684 685 FOR ${list} IN RANGE ${length} 686 @{words}= Split String ${lists}[${list}] ${SPACE} 687 Create Error Log ${words}[0] ${words}[1] ${words}[2] 688 END 689 690 # Create an additional error log to exceed max error logs limit. 691 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 692 693 # Delay for BMC to perform delete older error logs when log limit exceeds. 694 Sleep 10s 695 696 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 697 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 698 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 699 700 ${disk_usage_percentage}= Get Disk Usage For Error Logs 701 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 702 ${percent_diff}= Evaluate abs(${percent_diff}) 703 704 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 705 ... Evaluate ${percent_diff} <= 0.5 706 ... ELSE 707 ... Set Variable True 708 709 # Check PEL log count via peltool command and compare it with actual generated log count. 710 ${pel_records}= peltool -n 711 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 712 # Number of logs can be 80% of the total logs created after trimming. 713 ${expected_max_record}= Evaluate 3000 * 0.8 714 715 Run Keyword If ${trimmed_as_expected} == False 716 ... Should Be True ${no_pel_records} <= ${expected_max_record} 717 718 719 Create Error Log 720 [Documentation] Create an error log. 721 [Arguments] ${error_severity} ${error_creator} ${count} 722 723 # Description of argument(s): 724 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 725 # error_creator Name of error log's creator(E.g BMC, Host Boot) 726 # count Number of error logs to be generated. 727 728 FOR ${i} IN RANGE 0 ${count} 729 ${cmd}= Set Variable If 730 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 731 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 732 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 733 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 734 BMC Execute Command ${cmd} 735 END 736 737 738 Get Disk Usage For Error Logs 739 [Documentation] Get disk usage percentage for error logs. 740 741 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 742 743 ${usage_output}= Fetch From Left ${usage_output} \/ 744 745 # Covert disk usage unit from KB to MB. 746 ${usage_output}= Evaluate ${usage_output} / 1024 747 748 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 749 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 750 751 [return] ${usage_percent} 752 753 754 Get PEL Log IDs 755 [Documentation] Returns the list of PEL log IDs which contains given field's value. 756 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 757 758 # Description of argument(s): 759 # pel_section The section of PEL (e.g. Private Header, User Header). 760 # pel_field The PEL field (e.g. Event Severity, Event Type). 761 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 762 763 ${pel_ids}= Get PEL Log Via BMC CLI 764 @{pel_id_list}= Create List 765 766 FOR ${id} IN @{pel_ids} 767 ${pel_output}= Peltool -i ${id} 768 # Example of PEL output from "peltool -i <id>" command. 769 # [Private Header]: 770 # [Created at]: 08/24/1928 12:04:06 771 # [Created by]: 0x584D 772 # [Sub-section type]: 0 773 # [Entry Id]: 0x50000BB7 774 # [Platform Log Id]: 0x8200061D 775 # [CSSVER]: 776 # [Section Version]: 1 777 # [Creator Subsystem]: PHYP 778 # [BMC Event Log Id]: 341 779 # [Committed at]: 03/25/1920 12:06:22 780 # [User Header]: 781 # [Log Committed by]: 0x4552 782 # [Action Flags]: 783 # [0]: Report Externally 784 # [Subsystem]: I/O Subsystem 785 # [Event Type]: Miscellaneous, Informational Only 786 # [Sub-section type]: 0 787 # [Event Scope]: Entire Platform 788 # [Event Severity]: Informational Event 789 # [Host Transmission]: Not Sent 790 # [Section Version]: 1 791 792 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 793 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 794 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 795 END 796 Sort List ${pel_id_list} 797 798 [Return] ${pel_id_list} 799 800 801 Get PEL Log Via BMC CLI 802 [Documentation] Returns the list of PEL IDs using BMC CLI. 803 804 ${pel_records}= Peltool -l 805 ${ids}= Get Dictionary Keys ${pel_records} 806 Sort List ${ids} 807 808 [Return] ${ids} 809 810 811 Get PEL Field Value 812 [Documentation] Returns the value of given PEL's field. 813 [Arguments] ${pel_id} ${pel_section} ${pel_field} 814 815 # Description of argument(s): 816 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 817 # pel_section The section of PEL (e.g. Private Header, User Header) 818 # pel_field The PEL field (e.g. Event Severity, Event Type). 819 820 ${pel_output}= Peltool -i ${pel_id} 821 822 # Example of PEL output from "peltool -i <id>" command. 823 # [Private Header]: 824 # [Created at]: 08/24/1928 12:04:06 825 # [Created by]: 0x584D 826 # [Sub-section type]: 0 827 # [Entry Id]: 0x50000BB7 828 # [Platform Log Id]: 0x8200061D 829 # [CSSVER]: 830 # [Section Version]: 1 831 # [Creator Subsystem]: PHYP 832 # [BMC Event Log Id]: 341 833 # [Committed at]: 03/25/1920 12:06:22 834 # [User Header]: 835 # [Log Committed by]: 0x4552 836 # [Action Flags]: 837 # [0]: Report Externally 838 # [Subsystem]: I/O Subsystem 839 # [Event Type]: Miscellaneous, Informational Only 840 # [Sub-section type]: 0 841 # [Event Scope]: Entire Platform 842 # [Event Severity]: Informational Event 843 # [Host Transmission]: Not Sent 844 # [Section Version]: 1 845 846 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 847 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 848 849 [Return] ${pel_field_output} 850