1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17@{mandatory_pel_fields} Private Header User Header Primary SRC 18... Extended User Header Failing MTMS 19@{mandatory_Predictive_pel_fields} Private Header User Header Primary SRC 20... Extended User Header Failing MTMS User Data 21@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 22... Extended User Header User Data 23 24*** Test Cases *** 25 26Create Test PEL Log And Verify 27 [Documentation] Create PEL log using busctl command and verify via peltool. 28 [Tags] Create_Test_PEL_Log_And_Verify 29 30 Redfish Purge Event Log 31 Create Test PEL Log 32 ${pel_id}= Get PEL Log Via BMC CLI 33 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 34 35 36Verify PEL Log Details 37 [Documentation] Verify PEL log details via peltool. 38 [Tags] Verify_PEL_Log_Details 39 40 Redfish Purge Event Log 41 42 ${bmc_time1}= CLI Get BMC DateTime 43 Create Test PEL Log 44 ${bmc_time2}= CLI Get BMC DateTime 45 46 ${pel_records}= Peltool -l 47 48 # Example output from 'Peltool -l': 49 # pel_records: 50 # [0x50000012]: 51 # [CreatorID]: BMC 52 # [CompID]: 0x1000 53 # [PLID]: 0x50000012 54 # [Subsystem]: BMC Firmware 55 # [Message]: An application had an internal failure 56 # [SRC]: BD8D1002 57 # [Commit Time]: 03/02/2020 09:35:15 58 # [Sev]: Unrecoverable Error 59 60 ${ids}= Get Dictionary Keys ${pel_records} 61 ${id}= Get From List ${ids} 0 62 63 @{pel_fields}= Create List CreatorID Subsystem Message Sev 64 FOR ${field} IN @{pel_fields} 65 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 66 END 67 68 Valid Value pel_records['${id}']['PLID'] ['${id}'] 69 70 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 71 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 72 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 73 74 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 75 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 76 77 # Convert BMC and PEL time to epoch time before comparing. 78 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 79 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 80 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 81 82 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 83 84 85Verify Mandatory Sections Of Error Log PEL 86 [Documentation] Verify mandatory sections of error log PEL. 87 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 88 89 Create Test PEL Log 90 91 ${pel_ids}= Get PEL Log Via BMC CLI 92 ${pel_id}= Get From List ${pel_ids} -1 93 ${pel_output}= Peltool -i ${pel_id} 94 ${pel_sections}= Get Dictionary Keys ${pel_output} 95 96 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fields} 97 98 99Verify PEL Log Persistence After BMC Reboot 100 [Documentation] Verify PEL log persistence after BMC reboot. 101 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 102 103 Create Test PEL Log 104 ${pel_before_reboot}= Get PEL Log Via BMC CLI 105 106 Redfish OBMC Reboot (off) 107 ${pel_after_reboot}= Get PEL Log Via BMC CLI 108 109 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 110 111 112Verify PEL ID Numbering 113 [Documentation] Verify PEL ID numbering. 114 [Tags] Verify_PEL_ID_Numbering 115 116 Redfish Purge Event Log 117 Create Test PEL Log 118 Create Test PEL Log 119 120 ${pel_ids}= Get PEL Log Via BMC CLI 121 122 # Example of PEL IDs from PEL logs. 123 # [0x50000012]: <--- First PEL ID 124 # [CreatorID]: BMC 125 # [CompID]: 0x1000 126 # [PLID]: 0x50000012 127 # [Subsystem]: BMC Firmware 128 # [Message]: An application had an internal failure 129 # [SRC]: BD8D1002 130 # [Commit Time]: 03/02/2020 09:35:15 131 # [Sev]: Unrecoverable Error 132 # 133 # [0x50000013]: <--- Second PEL ID 134 # [CreatorID]: BMC 135 # [CompID]: 0x1000 136 # [PLID]: 0x50000013 137 # [Subsystem]: BMC Firmware 138 # [Message]: An application had an internal failure 139 # [SRC]: BD8D1002 140 # [Commit Time]: 03/02/2020 09:35:15 141 # [Sev]: Unrecoverable Error 142 143 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 144 145Verify Machine Type Model And Serial Number 146 [Documentation] Verify machine type model and serial number from PEL. 147 [Tags] Verify_Machine_Type_Model_And_Serial_Number 148 149 Create Test PEL Log 150 151 ${pel_ids}= Get PEL Log Via BMC CLI 152 ${id}= Get From List ${pel_ids} -1 153 154 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 155 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 156 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 157 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 158 159 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 160 # [Failing MTMS]: 161 # [Created by]: 0x2000 162 # [Machine Type Model]: 1234-ABC <---- Machine type 163 # [Section Version]: 1 164 # [Serial Number]: ABCDEFG <---- Serial number 165 # [Sub-section type]: 0 166 167 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 168 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 169 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 170 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 171 172 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 173 Valid Value pel_serial_number ['${redfish_serial_number}'] 174 175 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 176 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 177 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 178 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 179 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 180 181 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 182 Valid Value pel_serial_number ['${redfish_serial_number}'] 183 184 185Verify Host Off State From PEL 186 [Documentation] Verify Host off state from PEL. 187 [Tags] Verify_Host_Off_State_From_PEL 188 189 Redfish Power Off stack_mode=skip 190 Create Test PEL Log 191 192 ${pel_ids}= Get PEL Log Via BMC CLI 193 ${id}= Get From List ${pel_ids} -1 194 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 195 196 Valid Value pel_host_state ['Off'] 197 198 199Verify BMC Version From PEL 200 [Documentation] Verify BMC Version from PEL. 201 [Tags] Verify_BMC_Version_From_PEL 202 203 Create Test PEL Log 204 205 ${pel_ids}= Get PEL Log Via BMC CLI 206 ${id}= Get From List ${pel_ids} -1 207 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 208 209 ${bmc_version}= Get BMC Version 210 Valid Value bmc_version ['${bmc_version}'] 211 212 213Verify PEL Log After Host Poweron 214 [Documentation] Verify PEL log generation while booting host. 215 [Tags] Verify_PEL_Log_After_Host_Poweron 216 217 Redfish Power Off stack_mode=skip 218 Redfish Purge Event Log 219 Redfish Power On stack_mode=skip 220 221 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 222 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 223 224 # Get BMC created non-infomational error. 225 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 226 227 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 228 229 230Verify BMC Event Log ID 231 [Documentation] Verify BMC Event Log ID from PEL. 232 [Tags] Verify_BMC_Event_Log_ID 233 234 Redfish Purge Event Log 235 Create Test PEL Log 236 237 ${pel_ids}= Get PEL Log Via BMC CLI 238 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 239 240 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 241 # [Private Header]: 242 # [Created at]: 08/24/1928 12:04:06 243 # [Created by]: 0x584D 244 # [Sub-section type]: 0 245 # [Entry Id]: 0x50000BB7 246 # [Platform Log Id]: 0x8200061D 247 # [CSSVER]: 248 # [Section Version]: 1 249 # [Creator Subsystem]: PHYP 250 # [BMC Event Log Id]: 341 <---- BMC event log id value 251 # [Committed at]: 03/25/1920 12:06:22 252 253 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 254 255 # Example of redfish_event_logs output: 256 # redfish_event_logs: 257 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 258 # [Name]: System Event Log Entries 259 # [Members@odata.count]: 1 260 # [@odata.type]: #LogEntryCollection.LogEntryCollection 261 # [Description]: Collection of System Event Log Entries 262 # [Members]: 263 # [0]: 264 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 265 # [Name]: System Event Log Entry 266 # [Severity]: Critical 267 # [EntryType]: Event 268 # [Created]: 2020-04-02T07:25:13+00:00 269 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 270 # [Id]: 235 <----- Event log ID 271 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 272 273 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 274 275 276Verify FRU Callout 277 [Documentation] Verify FRU callout entries from PEL log. 278 [Tags] Verify_FRU_Callout 279 280 # Power off the system to avoid unnecessary error logs during test. 281 Redfish Power Off stack_mode=skip 282 Create Test PEL Log FRU Callout 283 284 ${pel_ids}= Get PEL Log Via BMC CLI 285 ${id}= Get From List ${pel_ids} -1 286 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 287 288 # Example of PEL Callout Section from "peltool -i <id>" command. 289 # [Callouts]: 290 # [0]: 291 # [FRU Type]: Normal Hardware FRU 292 # [Priority]: Mandatory, replace all with this type as a unit 293 # [Location Code]: U78DA.ND1.1234567-P0 294 # [Part Number]: F191014 295 # [CCIN]: 2E2D 296 # [Serial Number]: YL2E2D010000 297 # [Callout Count]: 1 298 299 Valid Value pel_callout_section['Callout Count'] ['1'] 300 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 301 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 302 303 # Verify Location Code field of PEL callout with motherboard's Location Code. 304 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 305 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 306 ... ${busctl_output[0].split('"')[1].strip('"')} 307 308 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 309 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 310 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 311 312 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 313 ${busctl_output}= BMC Execute Command 314 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 315 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 316 ... ${busctl_output[0].split('"')[1].strip('"')} 317 318 319Verify Procedure And Symbolic FRU Callout 320 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 321 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 322 323 Create Test PEL Log Procedure And Symbolic FRU Callout 324 325 ${pel_ids}= Get PEL Log Via BMC CLI 326 ${id}= Get From List ${pel_ids} -1 327 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 328 329 # Example of PEL Callout Section from "peltool -i <id>" command. 330 # [Callouts]: 331 # [0]: 332 # [Priority]: Mandatory, replace all with this type as a unit 333 # [Procedure Number]: BMCSP02 334 # [FRU Type]: Maintenance Procedure Required 335 # [1]: 336 # [Priority]: Medium Priority 337 # [Part Number]: SVCDOCS 338 # [FRU Type]: Symbolic FRU 339 # [Callout Count]: 2 340 341 Valid Value pel_callout_section['Callout Count'] ['2'] 342 343 # Verify procedural callout info. 344 345 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 346 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 347 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 348 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 349 350 # Verify procedural callout info. 351 352 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 353 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 354 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 355 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 356 357 358Verify PEL Log Entry For Event Log 359 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 360 [Tags] Verify_PEL_Log_Entry_For_Event_Log 361 362 Redfish Purge Event Log 363 364 # Create an unrecoverable error log. 365 Create Test PEL Log Unrecoverable Error 366 367 ${elog_entry}= Get Event Logs 368 # Example of Redfish event logs: 369 # elog_entry: 370 # [0]: 371 # [Message]: BD802003 event in subsystem: Platform Firmware 372 # [Created]: 2020-04-20T01:55:22+00:00 373 # [Id]: 1 374 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 375 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 376 # [EntryType]: Event 377 # [Severity]: Critical 378 # [Name]: System Event Log Entry 379 380 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 381 382 ${pel_records}= Peltool -l 383 # Example output from 'Peltool -l': 384 # pel_records: 385 # [0x50000023]: 386 # [SRC]: BD802003 387 # [CreatorID]: BMC 388 # [Message]: This is a test error 389 # [CompID]: 0x2000 390 # [PLID]: 0x500053D9 391 # [Commit Time]: 04/20/2020 01:55:22 392 # [Subsystem]: Platform Firmware 393 # [Sev]: Unrecoverable Error 394 395 ${ids}= Get Dictionary Keys ${pel_records} 396 ${id}= Get From List ${ids} 0 397 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 398 ... date_format=%m/%d/%Y %H:%M:%S 399 400 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 401 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 402 Should Contain ${elog_entry[0]['Message']} 403 ... ${pel_records['${id}']['SRC']} ignore_case=True 404 405 Should Be Equal ${redfish_log_time} ${pel_log_time} 406 407 408Verify Delete All PEL 409 [Documentation] Verify deleting all PEL logs. 410 [Tags] Verify_Delete_All_PEL 411 412 Create Test PEL Log 413 Create Test PEL Log 414 Peltool --delete-all False 415 416 ${pel_ids}= Get PEL Log Via BMC CLI 417 Should Be Empty ${pel_ids} 418 419 420Verify Informational Error Log 421 [Documentation] Create an informational error log and verify. 422 [Tags] Verify_Informational_Error_Log 423 424 Redfish Purge Event Log 425 # Create an informational error log. 426 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 427 ${pel_records}= Peltool -lfh 428 429 # An example of information error log data: 430 # { 431 # "0x500006A0": { 432 # "SRC": "BD8D1002", 433 # "Message": "An application had an internal failure", 434 # "PLID": "0x500006A0", 435 # "CreatorID": "BMC", 436 # "Subsystem": "BMC Firmware", 437 # "Commit Time": "10/14/2020 11:41:38", 438 # "Sev": "Informational Event", 439 # "CompID": "0x1000" 440 # } 441 # } 442 443 ${ids}= Get Dictionary Keys ${pel_records} 444 ${id}= Get From List ${ids} 0 445 Should Contain ${pel_records['${id}']['Sev']} Informational 446 447 448Verify Predictable Error Log 449 [Documentation] Create a predictive error and verify. 450 [Tags] Verify_Predictable_Error_Log 451 452 # Create a predictable error log. 453 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 454 ${pel_records}= Peltool -l 455 456 # An example of predictive error log data: 457 # { 458 # "0x5000069E": { 459 # "SRC": "BD8D1002", 460 # "Message": "An application had an internal failure", 461 # "PLID": "0x5000069E", 462 # "CreatorID": "BMC", 463 # "Subsystem": "BMC Firmware", 464 # "Commit Time": "10/14/2020 11:40:07", 465 # "Sev": "Predictive Error", 466 # "CompID": "0x1000" 467 # } 468 # } 469 470 ${pel_ids}= Get PEL Log Via BMC CLI 471 ${id}= Get From List ${pel_ids} -1 472 Should Contain ${pel_records['${id}']['Sev']} Predictive 473 474 475Verify Unrecoverable Error Log 476 [Documentation] Create an unrecoverable error and verify. 477 [Tags] Verify_Unrecoverable_Error_Log 478 479 # Create an internal failure error log. 480 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 481 ${pel_records}= Peltool -l 482 483 # An example of unrecoverable error log data: 484 # { 485 # "0x50000CC5": { 486 # "SRC": "BD8D1002", 487 # "Message": "An application had an internal failure", 488 # "PLID": "0x50000CC5", 489 # "CreatorID": "BMC", 490 # "Subsystem": "BMC Firmware", 491 # "Commit Time": "04/01/2020 16:44:55", 492 # "Sev": "Unrecoverable Error", 493 # "CompID": "0x1000" 494 # } 495 # } 496 497 ${pel_ids}= Get PEL Log Via BMC CLI 498 ${id}= Get From List ${pel_ids} -1 499 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 500 501 502Verify Error Logging Rotation Policy 503 [Documentation] Verify error logging rotation policy. 504 [Tags] Verify_Error_Logging_Rotation_Policy 505 [Template] Error Logging Rotation Policy 506 507 # Error logs to be created % of total logging space when error 508 # log exceeds max limit. 509 Informational BMC 3000 15 510 Predictive BMC 3000 30 511 Unrecoverable BMC 3000 30 512 Informational BMC 1500, Predictive BMC 1500 45 513 Informational BMC 1500, Unrecoverable BMC 1500 45 514 Unrecoverable BMC 1500, Predictive BMC 1500 30 515 516 517Verify Error Logging Rotation Policy With All Types Of Errors 518 [Documentation] Verify error logging rotation policy with all types of errors. 519 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 520 [Template] Error Logging Rotation Policy 521 522 # Error logs to be created % of total logging space when error 523 # log exceeds max limit. 524 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 525 526 527Verify Error Logging Rotation Policy With HOST Error Logs 528 [Documentation] Verify error logging rotation policy for non bmc error logs. 529 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 530 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 531 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 532 ... /tmp/FILE_HOST_INFORMATIONAL 533 [Template] Error Logging Rotation Policy 534 535 # Error logs to be created % of total logging space when error 536 # log exceeds max limit. 537 Informational HOST 3000 15 538 Unrecoverable HOST 3000 30 539 Informational HOST 1500, Informational BMC 1500 30 540 Informational HOST 1500, Unrecoverable BMC 1500 45 541 Unrecoverable HOST 1500, Informational BMC 1500 45 542 Unrecoverable HOST 1500, Predictive BMC 1500 60 543 544 545Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 546 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 547 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 548 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 549 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 550 [Template] Error Logging Rotation Policy 551 552 # Error logs to be created % of total logging space when error 553 # log exceeds max limit. 554 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 555 556 557Verify Old Logs Are Deleted When Count Crosses Max 558 [Documentation] Verify that when the count crosses max, older logs are deleted. 559 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 560 561 Redfish Purge Event Log 562 # Create 3000 error logs. 563 FOR ${count} IN RANGE ${3000} 564 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 565 END 566 567 # Retrieve the IDs of the logs. 568 ${pel_ids}= Get PEL Log Via BMC CLI 569 ${1st_id}= Get From List ${pel_ids} 0 570 ${3000th_id}= Get From List ${pel_ids} 2999 571 572 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 573 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 574 575 # Wait few seconds for error logs rotation to complete. 576 Sleep 10s 577 578 # Now verify that first log is no more available but the 3000th is available. 579 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 580 Should Be True '${status}' == 'FAIL' 581 Should Contain ${output} PEL not found 582 583 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 584 Should Be True '${status}' == 'PASS' 585 Should Not Contain ${output} PEL not found 586 587 588Verify Reverse Order Of PEL Logs 589 [Documentation] Verify PEL command to output PEL logs in reverse order. 590 [Tags] Verify_Reverse_Order_Of_PEL_Logs 591 592 Redfish Purge Event Log 593 594 # Below commands create unrecoverable error log at first and then the predictable error. 595 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 596 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 597 598 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 599 ${pel_records}= peltool -lr 600 601 # It is found that, variables like dictionary always keep items in sorted order that makes 602 # this verification not possible, hence json is used to keep the items original order. 603 ${pel_records}= Convert To String ${pel_records} 604 ${json_string}= Replace String ${pel_records} ' " 605 ${json_object}= Evaluate json.loads('''${json_string}''') json 606 607 ${list}= Convert To List ${json_object} 608 609 ${id1}= Get From List ${list} 0 610 ${id1}= Convert To Integer ${id1} 611 ${id2}= Get From List ${list} 1 612 ${id2}= Convert To Integer ${id2} 613 614 Should Be True ${id1} > ${id2} 615 616 617Verify Total PEL Count 618 [Documentation] Verify total PEL count returned by peltool command. 619 [Tags] Verify_Total_PEL_Count 620 621 # Initially remove all logs. 622 Redfish Purge Event Log 623 624 # Generate a random number between 1-20. 625 ${random}= Evaluate random.randint(1, 20) modules=random 626 627 # Generate predictive error log multiple times. 628 FOR ${count} IN RANGE 0 ${random} 629 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 630 END 631 632 # Check PEL log count via peltool command and compare it with actual generated log count. 633 ${pel_records}= peltool -n 634 635 Should Be Equal ${pel_records['Number of PELs found']} ${random} 636 637 638Verify Listing Information Error 639 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 640 [Tags] Verify_Listing_Information_Error 641 642 # Initially remove all logs. 643 Redfish Purge Event Log 644 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 645 646 # Generate informational logs and verify that it would not get listed by peltool's list command. 647 ${pel_records}= peltool -l 648 ${ids}= Get Dictionary Keys ${pel_records} 649 Should Be Empty ${ids} 650 651 # Verify that information logs get listed using peltool's list command with -lfh option. 652 ${pel_records}= peltool -lfh 653 ${ids}= Get Dictionary Keys ${pel_records} 654 Should Not Be Empty ${ids} 655 ${id}= Get From List ${ids} 0 656 Should Contain ${pel_records['${id}']['Sev']} Informational 657 658 659Verify PEL Delete 660 [Documentation] Verify that peltool command can delete PEL log based on id. 661 [Tags] Verify_PEL_Delete 662 663 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 664 ${pel_ids}= Get PEL Log Via BMC CLI 665 ${id}= Get From List ${pel_ids} -1 666 Peltool -d ${id} False 667 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 668 669 670Verify Mandatory Fields For Predictive Error 671 [Documentation] Verify mandatory fields of predictive errors from pel information. 672 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 673 674 # Inject predictive error. 675 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 676 677 ${pel_ids}= Get PEL Log Via BMC CLI 678 ${pel_id}= Get From List ${pel_ids} -1 679 ${pel_output}= Peltool -i ${pel_id} 680 # Get all fields in predictive error log. 681 ${pel_sections}= Get Dictionary Keys ${pel_output} 682 683 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fields} 684 685 686Verify Mandatory Fields For Informational Error 687 [Documentation] Verify mandatory fields of informational error from pel information. 688 [Tags] Verify_Mandatory_Fields_For_Informational_Error 689 690 # Inject informational error. 691 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 692 ${pel_records}= Peltool -lfh 693 694 ${ids}= Get Dictionary Keys ${pel_records} 695 ${pel_id}= Get From List ${ids} -1 696 ${pel_output}= Peltool -i ${pel_id} 697 698 # Get all fields in the informational error log. 699 ${pel_sections}= Get Dictionary Keys ${pel_output} 700 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 701 ${contains}= Evaluate "${section}" in "${pel_sections}" 702 Should Be True ${contains} 703 END 704 705 706Verify PEL Log Not Offloaded To Host 707 [Documentation] Verify host transmission state of not offloaded PEL logs. 708 [Tags] Verify_PEL_Log_Not_Offloaded_To_Host 709 [Template] Verify PEL Transmission To Host 710 711 # error_type host_state expected_transmission_state 712 informational_error Off Not Sent 713 unrecoverable_error Off Not Sent 714 predictive_error Off Not Sent 715 716 717Verify PEL Log Offloaded To Host 718 [Documentation] Verify host transmission state of offloaded PEL logs. 719 [Tags] Verify_PEL_Log_Offloaded_To_Host 720 [Template] Verify PEL Transmission To Host 721 722 # error_type host_state expected_transmission_state 723 unrecoverable_error On Acked 724 predictive_error On Acked 725 informational_error On Acked 726 727 728*** Keywords *** 729 730Error Logging Rotation Policy 731 [Documentation] Verify that when maximum log limit is reached, given error logging type 732 ... are deleted when reached their max allocated space. 733 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 734 735 # Description of argument(s): 736 # error_log Error logs to be created (E.g. Informational BMC 3000 737 # stands for BMC created 3000 informational error logs). 738 # max_allocated_space_percentage The maximum percentage of disk usage for given error 739 # log type when maximum count/log size is reached. 740 # The maximum error log count is 3000. 741 742 Redfish.Login 743 744 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 745 # in BMC during the test. 746 Redfish Purge Event Log 747 748 @{lists}= Split String ${error_log_type} ,${SPACE} 749 750 ${length}= Get Length ${lists} 751 752 FOR ${list} IN RANGE ${length} 753 @{words}= Split String ${lists}[${list}] ${SPACE} 754 Create Error Log ${words}[0] ${words}[1] ${words}[2] 755 END 756 757 # Create an additional error log to exceed max error logs limit. 758 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 759 760 # Delay for BMC to perform delete older error logs when log limit exceeds. 761 Sleep 10s 762 763 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 764 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 765 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 766 767 ${disk_usage_percentage}= Get Disk Usage For Error Logs 768 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 769 ${percent_diff}= Evaluate abs(${percent_diff}) 770 771 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 772 ... Evaluate ${percent_diff} <= 0.5 773 ... ELSE 774 ... Set Variable True 775 776 # Check PEL log count via peltool command and compare it with actual generated log count. 777 ${pel_records}= peltool -n 778 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 779 # Number of logs can be 80% of the total logs created after trimming. 780 ${expected_max_record}= Evaluate 3000 * 0.8 781 782 Run Keyword If ${trimmed_as_expected} == False 783 ... Should Be True ${no_pel_records} <= ${expected_max_record} 784 785 786Create Error Log 787 [Documentation] Create an error log. 788 [Arguments] ${error_severity} ${error_creator} ${count} 789 790 # Description of argument(s): 791 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 792 # error_creator Name of error log's creator(E.g BMC, Host Boot) 793 # count Number of error logs to be generated. 794 795 FOR ${i} IN RANGE 0 ${count} 796 ${cmd}= Set Variable If 797 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 798 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 799 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 800 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 801 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 802 BMC Execute Command ${cmd} 803 END 804 805 806Get Disk Usage For Error Logs 807 [Documentation] Get disk usage percentage for error logs. 808 809 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 810 811 ${usage_output}= Fetch From Left ${usage_output} \/ 812 813 # Convert disk usage unit from KB to MB. 814 ${usage_output}= Evaluate ${usage_output} / 1024 815 816 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 817 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 818 819 [return] ${usage_percent} 820 821 822Get PEL Log IDs 823 [Documentation] Returns the list of PEL log IDs which contains given field's value. 824 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 825 826 # Description of argument(s): 827 # pel_section The section of PEL (e.g. Private Header, User Header). 828 # pel_field The PEL field (e.g. Event Severity, Event Type). 829 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 830 831 ${pel_ids}= Get PEL Log Via BMC CLI 832 @{pel_id_list}= Create List 833 834 FOR ${id} IN @{pel_ids} 835 ${pel_output}= Peltool -i ${id} 836 # Example of PEL output from "peltool -i <id>" command. 837 # [Private Header]: 838 # [Created at]: 08/24/1928 12:04:06 839 # [Created by]: 0x584D 840 # [Sub-section type]: 0 841 # [Entry Id]: 0x50000BB7 842 # [Platform Log Id]: 0x8200061D 843 # [CSSVER]: 844 # [Section Version]: 1 845 # [Creator Subsystem]: PHYP 846 # [BMC Event Log Id]: 341 847 # [Committed at]: 03/25/1920 12:06:22 848 # [User Header]: 849 # [Log Committed by]: 0x4552 850 # [Action Flags]: 851 # [0]: Report Externally 852 # [Subsystem]: I/O Subsystem 853 # [Event Type]: Miscellaneous, Informational Only 854 # [Sub-section type]: 0 855 # [Event Scope]: Entire Platform 856 # [Event Severity]: Informational Event 857 # [Host Transmission]: Not Sent 858 # [Section Version]: 1 859 860 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 861 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 862 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 863 END 864 Sort List ${pel_id_list} 865 866 [Return] ${pel_id_list} 867 868 869Get PEL Log Via BMC CLI 870 [Documentation] Returns the list of PEL IDs using BMC CLI. 871 872 ${pel_records}= Peltool -l 873 ${ids}= Get Dictionary Keys ${pel_records} 874 Sort List ${ids} 875 876 [Return] ${ids} 877 878 879Get PEL Field Value 880 [Documentation] Returns the value of given PEL's field. 881 [Arguments] ${pel_id} ${pel_section} ${pel_field} 882 883 # Description of argument(s): 884 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 885 # pel_section The section of PEL (e.g. Private Header, User Header) 886 # pel_field The PEL field (e.g. Event Severity, Event Type). 887 888 ${pel_output}= Peltool -i ${pel_id} 889 890 # Example of PEL output from "peltool -i <id>" command. 891 # [Private Header]: 892 # [Created at]: 08/24/1928 12:04:06 893 # [Created by]: 0x584D 894 # [Sub-section type]: 0 895 # [Entry Id]: 0x50000BB7 896 # [Platform Log Id]: 0x8200061D 897 # [CSSVER]: 898 # [Section Version]: 1 899 # [Creator Subsystem]: PHYP 900 # [BMC Event Log Id]: 341 901 # [Committed at]: 03/25/1920 12:06:22 902 # [User Header]: 903 # [Log Committed by]: 0x4552 904 # [Action Flags]: 905 # [0]: Report Externally 906 # [Subsystem]: I/O Subsystem 907 # [Event Type]: Miscellaneous, Informational Only 908 # [Sub-section type]: 0 909 # [Event Scope]: Entire Platform 910 # [Event Severity]: Informational Event 911 # [Host Transmission]: Not Sent 912 # [Section Version]: 1 913 914 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 915 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 916 917 [Return] ${pel_field_output} 918 919 920Verify PEL Transmission To Host 921 [Documentation] Inject PEL log of given type and verify its host transmission state. 922 [Arguments] ${error_type} ${host_state} ${expected_transmission_state} 923 924 # Description of argument(s): 925 # error_type Type of error log to be injected. 926 # host_state Host state which is required before error log injection. 927 # expected_transmission_state Expected host transmission state of PEL log. 928 929 # Get system in required state. 930 Run Keyword If '${host_state}' == 'Off' 931 ... Redfish Power Off stack_mode=skip 932 ... ELSE IF '${host_state}' == 'On' 933 ... Redfish Power On stack_mode=skip 934 935 Redfish Purge Event Log 936 937 # Inject required error log. 938 Run Keyword If "${error_type}" == "informational_error" 939 ... BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 940 ... ELSE IF "${error_type}" == "unrecoverable_error" 941 ... BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 942 ... ELSE IF "${error_type}" == "predictive_error" 943 ... BMC Execute Command ${CMD_PREDICTIVE_ERROR} 944 945 ${pel_records}= Get Pel Data From Bmc True True 946 ${ids}= Get Dictionary Keys ${pel_records} 947 ${pel_id}= Get From List ${ids} -1 948 949 # Check host transmission state for the cases where PEL is 950 # expected to be offloaded to HOST. 951 Run Keyword If "${expected_transmission_state}" == "Acked" 952 ... Wait Until Keyword Succeeds 2 min 10 sec 953 ... Check If PEL Transmission State Is Expected ${pel_id} Acked 954 955 # Adding delay before checking host transmission for the cases where PEL is 956 # not expected to be offloaded to HOST. 957 Run Keyword If "${expected_transmission_state}" == "Not sent" 958 ... Run Keywords Sleep 120s AND 959 ... Check If PEL Transmission State Is Expected ${pel_id} Not sent 960 961 962Check If PEL Transmission State Is Expected 963 [Documentation] Check if PEL's host transmission state is matching expected state. 964 [Arguments] ${pel_id} ${expected_transmission_state} 965 966 # Description of argument(s): 967 # expected_transmission_state Expected transmission state of PEL log. 968 969 # Check the transmission details in the error log. 970 ${host_transmission}= Get PEL Field Value ${pel_id} User Header Host Transmission 971 Should Be Equal ${host_transmission} ${expected_transmission_state} 972