1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17@{mandatory_pel_fields} Private Header User Header Primary SRC 18... Extended User Header Failing MTMS 19@{mandatory_Predictive_pel_fields} Private Header User Header Primary SRC 20... Extended User Header Failing MTMS User Data 21@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 22... Extended User Header User Data 23 24*** Test Cases *** 25 26Create Test PEL Log And Verify 27 [Documentation] Create PEL log using busctl command and verify via peltool. 28 [Tags] Create_Test_PEL_Log_And_Verify 29 30 Redfish Purge Event Log 31 Create Test PEL Log 32 ${pel_id}= Get PEL Log Via BMC CLI 33 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 34 35 36Verify PEL Log Details 37 [Documentation] Verify PEL log details via peltool. 38 [Tags] Verify_PEL_Log_Details 39 40 Redfish Purge Event Log 41 42 ${bmc_time1}= CLI Get BMC DateTime 43 Create Test PEL Log 44 ${bmc_time2}= CLI Get BMC DateTime 45 46 ${pel_records}= Peltool -l 47 48 # Example output from 'Peltool -l': 49 # pel_records: 50 # [0x50000012]: 51 # [CreatorID]: BMC 52 # [CompID]: 0x1000 53 # [PLID]: 0x50000012 54 # [Subsystem]: BMC Firmware 55 # [Message]: An application had an internal failure 56 # [SRC]: BD8D1002 57 # [Commit Time]: 03/02/2020 09:35:15 58 # [Sev]: Unrecoverable Error 59 60 ${ids}= Get Dictionary Keys ${pel_records} 61 ${id}= Get From List ${ids} 0 62 63 @{pel_fields}= Create List CreatorID Subsystem Message Sev 64 FOR ${field} IN @{pel_fields} 65 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 66 END 67 68 Valid Value pel_records['${id}']['PLID'] ['${id}'] 69 70 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 71 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 72 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 73 74 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 75 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 76 77 # Convert BMC and PEL time to epoch time before comparing. 78 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 79 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 80 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 81 82 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 83 84 85Verify Mandatory Sections Of Error Log PEL 86 [Documentation] Verify mandatory sections of error log PEL. 87 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 88 89 Create Test PEL Log 90 91 ${pel_ids}= Get PEL Log Via BMC CLI 92 ${pel_id}= Get From List ${pel_ids} -1 93 ${pel_output}= Peltool -i ${pel_id} 94 ${pel_sections}= Get Dictionary Keys ${pel_output} 95 96 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fields} 97 98 99Verify PEL Log Persistence After BMC Reboot 100 [Documentation] Verify PEL log persistence after BMC reboot. 101 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 102 103 Create Test PEL Log 104 ${pel_before_reboot}= Get PEL Log Via BMC CLI 105 106 Redfish OBMC Reboot (off) 107 ${pel_after_reboot}= Get PEL Log Via BMC CLI 108 109 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 110 111 112Verify PEL ID Numbering 113 [Documentation] Verify PEL ID numbering. 114 [Tags] Verify_PEL_ID_Numbering 115 116 Redfish Purge Event Log 117 Create Test PEL Log 118 Create Test PEL Log 119 120 ${pel_ids}= Get PEL Log Via BMC CLI 121 122 # Example of PEL IDs from PEL logs. 123 # [0x50000012]: <--- First PEL ID 124 # [CreatorID]: BMC 125 # [CompID]: 0x1000 126 # [PLID]: 0x50000012 127 # [Subsystem]: BMC Firmware 128 # [Message]: An application had an internal failure 129 # [SRC]: BD8D1002 130 # [Commit Time]: 03/02/2020 09:35:15 131 # [Sev]: Unrecoverable Error 132 # 133 # [0x50000013]: <--- Second PEL ID 134 # [CreatorID]: BMC 135 # [CompID]: 0x1000 136 # [PLID]: 0x50000013 137 # [Subsystem]: BMC Firmware 138 # [Message]: An application had an internal failure 139 # [SRC]: BD8D1002 140 # [Commit Time]: 03/02/2020 09:35:15 141 # [Sev]: Unrecoverable Error 142 143 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 144 145Verify Machine Type Model And Serial Number 146 [Documentation] Verify machine type model and serial number from PEL. 147 [Tags] Verify_Machine_Type_Model_And_Serial_Number 148 149 Create Test PEL Log 150 151 ${pel_ids}= Get PEL Log Via BMC CLI 152 ${id}= Get From List ${pel_ids} -1 153 154 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 155 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 156 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 157 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 158 159 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 160 # [Failing MTMS]: 161 # [Created by]: 0x2000 162 # [Machine Type Model]: 1234-ABC <---- Machine type 163 # [Section Version]: 1 164 # [Serial Number]: ABCDEFG <---- Serial number 165 # [Sub-section type]: 0 166 167 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 168 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 169 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 170 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 171 172 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 173 Valid Value pel_serial_number ['${redfish_serial_number}'] 174 175 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 176 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 177 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 178 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 179 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 180 181 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 182 Valid Value pel_serial_number ['${redfish_serial_number}'] 183 184 185Verify Host Off State From PEL 186 [Documentation] Verify Host off state from PEL. 187 [Tags] Verify_Host_Off_State_From_PEL 188 189 Redfish Power Off stack_mode=skip 190 Create Test PEL Log 191 192 ${pel_ids}= Get PEL Log Via BMC CLI 193 ${id}= Get From List ${pel_ids} -1 194 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 195 196 Valid Value pel_host_state ['Off'] 197 198 199Verify Host On State From PEL 200 [Documentation] Verify Host on state from PEL. 201 [Tags] Verify_Host_On_State_From_PEL 202 203 Redfish Power On stack_mode=skip 204 Create Test PEL Log 205 206 ${pel_ids}= Get PEL Log Via BMC CLI 207 ${id}= Get From List ${pel_ids} -1 208 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 209 210 Valid Value pel_host_state ['Running'] 211 212 213Verify BMC Version From PEL 214 [Documentation] Verify BMC Version from PEL. 215 [Tags] Verify_BMC_Version_From_PEL 216 217 Create Test PEL Log 218 219 ${pel_ids}= Get PEL Log Via BMC CLI 220 ${id}= Get From List ${pel_ids} -1 221 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 222 223 ${bmc_version}= Get BMC Version 224 Valid Value bmc_version ['${bmc_version}'] 225 226 227Verify PEL Log After Host Poweron 228 [Documentation] Verify PEL log generation while booting host. 229 [Tags] Verify_PEL_Log_After_Host_Poweron 230 231 Redfish Power Off stack_mode=skip 232 Redfish Purge Event Log 233 Redfish Power On stack_mode=skip 234 235 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 236 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 237 238 # Get BMC created non-infomational error. 239 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 240 241 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 242 243 244Verify BMC Event Log ID 245 [Documentation] Verify BMC Event Log ID from PEL. 246 [Tags] Verify_BMC_Event_Log_ID 247 248 Redfish Purge Event Log 249 Create Test PEL Log 250 251 ${pel_ids}= Get PEL Log Via BMC CLI 252 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 253 254 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 255 # [Private Header]: 256 # [Created at]: 08/24/1928 12:04:06 257 # [Created by]: 0x584D 258 # [Sub-section type]: 0 259 # [Entry Id]: 0x50000BB7 260 # [Platform Log Id]: 0x8200061D 261 # [CSSVER]: 262 # [Section Version]: 1 263 # [Creator Subsystem]: PHYP 264 # [BMC Event Log Id]: 341 <---- BMC event log id value 265 # [Committed at]: 03/25/1920 12:06:22 266 267 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 268 269 # Example of redfish_event_logs output: 270 # redfish_event_logs: 271 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 272 # [Name]: System Event Log Entries 273 # [Members@odata.count]: 1 274 # [@odata.type]: #LogEntryCollection.LogEntryCollection 275 # [Description]: Collection of System Event Log Entries 276 # [Members]: 277 # [0]: 278 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 279 # [Name]: System Event Log Entry 280 # [Severity]: Critical 281 # [EntryType]: Event 282 # [Created]: 2020-04-02T07:25:13+00:00 283 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 284 # [Id]: 235 <----- Event log ID 285 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 286 287 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 288 289 290Verify FRU Callout 291 [Documentation] Verify FRU callout entries from PEL log. 292 [Tags] Verify_FRU_Callout 293 294 # Power off the system to avoid unnecessary error logs during test. 295 Redfish Power Off stack_mode=skip 296 Create Test PEL Log FRU Callout 297 298 ${pel_ids}= Get PEL Log Via BMC CLI 299 ${id}= Get From List ${pel_ids} -1 300 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 301 302 # Example of PEL Callout Section from "peltool -i <id>" command. 303 # [Callouts]: 304 # [0]: 305 # [FRU Type]: Normal Hardware FRU 306 # [Priority]: Mandatory, replace all with this type as a unit 307 # [Location Code]: U78DA.ND1.1234567-P0 308 # [Part Number]: F191014 309 # [CCIN]: 2E2D 310 # [Serial Number]: YL2E2D010000 311 # [Callout Count]: 1 312 313 Valid Value pel_callout_section['Callout Count'] ['1'] 314 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 315 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 316 317 # Verify Location Code field of PEL callout with motherboard's Location Code. 318 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 319 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 320 ... ${busctl_output[0].split('"')[1].strip('"')} 321 322 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 323 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 324 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 325 326 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 327 ${busctl_output}= BMC Execute Command 328 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 329 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 330 ... ${busctl_output[0].split('"')[1].strip('"')} 331 332 333Verify Procedure And Symbolic FRU Callout 334 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 335 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 336 337 Create Test PEL Log Procedure And Symbolic FRU Callout 338 339 ${pel_ids}= Get PEL Log Via BMC CLI 340 ${id}= Get From List ${pel_ids} -1 341 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 342 343 # Example of PEL Callout Section from "peltool -i <id>" command. 344 # [Callouts]: 345 # [0]: 346 # [Priority]: Mandatory, replace all with this type as a unit 347 # [Procedure Number]: BMCSP02 348 # [FRU Type]: Maintenance Procedure Required 349 # [1]: 350 # [Priority]: Medium Priority 351 # [Part Number]: SVCDOCS 352 # [FRU Type]: Symbolic FRU 353 # [Callout Count]: 2 354 355 Valid Value pel_callout_section['Callout Count'] ['2'] 356 357 # Verify procedural callout info. 358 359 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 360 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 361 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 362 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 363 364 # Verify procedural callout info. 365 366 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 367 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 368 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 369 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 370 371 372Verify PEL Log Entry For Event Log 373 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 374 [Tags] Verify_PEL_Log_Entry_For_Event_Log 375 376 Redfish Purge Event Log 377 378 # Create an unrecoverable error log. 379 Create Test PEL Log Unrecoverable Error 380 381 ${elog_entry}= Get Event Logs 382 # Example of Redfish event logs: 383 # elog_entry: 384 # [0]: 385 # [Message]: BD802003 event in subsystem: Platform Firmware 386 # [Created]: 2020-04-20T01:55:22+00:00 387 # [Id]: 1 388 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 389 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 390 # [EntryType]: Event 391 # [Severity]: Critical 392 # [Name]: System Event Log Entry 393 394 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 395 396 ${pel_records}= Peltool -l 397 # Example output from 'Peltool -l': 398 # pel_records: 399 # [0x50000023]: 400 # [SRC]: BD802003 401 # [CreatorID]: BMC 402 # [Message]: This is a test error 403 # [CompID]: 0x2000 404 # [PLID]: 0x500053D9 405 # [Commit Time]: 04/20/2020 01:55:22 406 # [Subsystem]: Platform Firmware 407 # [Sev]: Unrecoverable Error 408 409 ${ids}= Get Dictionary Keys ${pel_records} 410 ${id}= Get From List ${ids} 0 411 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 412 ... date_format=%m/%d/%Y %H:%M:%S 413 414 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 415 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 416 Should Contain ${elog_entry[0]['Message']} 417 ... ${pel_records['${id}']['SRC']} ignore_case=True 418 419 Should Be Equal ${redfish_log_time} ${pel_log_time} 420 421 422Verify Delete All PEL 423 [Documentation] Verify deleting all PEL logs. 424 [Tags] Verify_Delete_All_PEL 425 426 Create Test PEL Log 427 Create Test PEL Log 428 Peltool --delete-all False 429 430 ${pel_ids}= Get PEL Log Via BMC CLI 431 Should Be Empty ${pel_ids} 432 433 434Verify Informational Error Log 435 [Documentation] Create an informational error log and verify. 436 [Tags] Verify_Informational_Error_Log 437 438 Redfish Purge Event Log 439 # Create an informational error log. 440 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 441 ${pel_records}= Peltool -lfh 442 443 # An example of information error log data: 444 # { 445 # "0x500006A0": { 446 # "SRC": "BD8D1002", 447 # "Message": "An application had an internal failure", 448 # "PLID": "0x500006A0", 449 # "CreatorID": "BMC", 450 # "Subsystem": "BMC Firmware", 451 # "Commit Time": "10/14/2020 11:41:38", 452 # "Sev": "Informational Event", 453 # "CompID": "0x1000" 454 # } 455 # } 456 457 ${ids}= Get Dictionary Keys ${pel_records} 458 ${id}= Get From List ${ids} 0 459 Should Contain ${pel_records['${id}']['Sev']} Informational 460 461 462Verify Predictable Error Log 463 [Documentation] Create a predictive error and verify. 464 [Tags] Verify_Predictable_Error_Log 465 466 # Create a predictable error log. 467 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 468 ${pel_records}= Peltool -l 469 470 # An example of predictive error log data: 471 # { 472 # "0x5000069E": { 473 # "SRC": "BD8D1002", 474 # "Message": "An application had an internal failure", 475 # "PLID": "0x5000069E", 476 # "CreatorID": "BMC", 477 # "Subsystem": "BMC Firmware", 478 # "Commit Time": "10/14/2020 11:40:07", 479 # "Sev": "Predictive Error", 480 # "CompID": "0x1000" 481 # } 482 # } 483 484 ${pel_ids}= Get PEL Log Via BMC CLI 485 ${id}= Get From List ${pel_ids} -1 486 Should Contain ${pel_records['${id}']['Sev']} Predictive 487 488 489Verify Unrecoverable Error Log 490 [Documentation] Create an unrecoverable error and verify. 491 [Tags] Verify_Unrecoverable_Error_Log 492 493 # Create an internal failure error log. 494 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 495 ${pel_records}= Peltool -l 496 497 # An example of unrecoverable error log data: 498 # { 499 # "0x50000CC5": { 500 # "SRC": "BD8D1002", 501 # "Message": "An application had an internal failure", 502 # "PLID": "0x50000CC5", 503 # "CreatorID": "BMC", 504 # "Subsystem": "BMC Firmware", 505 # "Commit Time": "04/01/2020 16:44:55", 506 # "Sev": "Unrecoverable Error", 507 # "CompID": "0x1000" 508 # } 509 # } 510 511 ${pel_ids}= Get PEL Log Via BMC CLI 512 ${id}= Get From List ${pel_ids} -1 513 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 514 515 516Verify Error Logging Rotation Policy 517 [Documentation] Verify error logging rotation policy. 518 [Tags] Verify_Error_Logging_Rotation_Policy 519 [Template] Error Logging Rotation Policy 520 521 # Error logs to be created % of total logging space when error 522 # log exceeds max limit. 523 Informational BMC 3000 15 524 Predictive BMC 3000 30 525 Unrecoverable BMC 3000 30 526 Informational BMC 1500, Predictive BMC 1500 45 527 Informational BMC 1500, Unrecoverable BMC 1500 45 528 Unrecoverable BMC 1500, Predictive BMC 1500 30 529 530 531Verify Error Logging Rotation Policy With All Types Of Errors 532 [Documentation] Verify error logging rotation policy with all types of errors. 533 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 534 [Template] Error Logging Rotation Policy 535 536 # Error logs to be created % of total logging space when error 537 # log exceeds max limit. 538 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 539 540 541Verify Error Logging Rotation Policy With HOST Error Logs 542 [Documentation] Verify error logging rotation policy for non bmc error logs. 543 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 544 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 545 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 546 ... /tmp/FILE_HOST_INFORMATIONAL 547 [Template] Error Logging Rotation Policy 548 549 # Error logs to be created % of total logging space when error 550 # log exceeds max limit. 551 Informational HOST 3000 15 552 Unrecoverable HOST 3000 30 553 Informational HOST 1500, Informational BMC 1500 30 554 Informational HOST 1500, Unrecoverable BMC 1500 45 555 Unrecoverable HOST 1500, Informational BMC 1500 45 556 Unrecoverable HOST 1500, Predictive BMC 1500 60 557 558 559Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 560 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 561 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 562 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 563 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 564 [Template] Error Logging Rotation Policy 565 566 # Error logs to be created % of total logging space when error 567 # log exceeds max limit. 568 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 569 570 571Verify Old Logs Are Deleted When Count Crosses Max 572 [Documentation] Verify that when the count crosses max, older logs are deleted. 573 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 574 575 Redfish Purge Event Log 576 # Create 3000 error logs. 577 FOR ${count} IN RANGE ${3000} 578 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 579 END 580 581 # Retrieve the IDs of the logs. 582 ${pel_ids}= Get PEL Log Via BMC CLI 583 ${1st_id}= Get From List ${pel_ids} 0 584 ${3000th_id}= Get From List ${pel_ids} 2999 585 586 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 587 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 588 589 # Wait few seconds for error logs rotation to complete. 590 Sleep 10s 591 592 # Now verify that first log is no more available but the 3000th is available. 593 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 594 Should Be True '${status}' == 'FAIL' 595 Should Contain ${output} PEL not found 596 597 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 598 Should Be True '${status}' == 'PASS' 599 Should Not Contain ${output} PEL not found 600 601 602Verify Reverse Order Of PEL Logs 603 [Documentation] Verify PEL command to output PEL logs in reverse order. 604 [Tags] Verify_Reverse_Order_Of_PEL_Logs 605 606 Redfish Purge Event Log 607 608 # Below commands create unrecoverable error log at first and then the predictable error. 609 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 610 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 611 612 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 613 ${pel_records}= peltool -lr 614 615 # It is found that, variables like dictionary always keep items in sorted order that makes 616 # this verification not possible, hence json is used to keep the items original order. 617 ${pel_records}= Convert To String ${pel_records} 618 ${json_string}= Replace String ${pel_records} ' " 619 ${json_object}= Evaluate json.loads('''${json_string}''') json 620 621 ${list}= Convert To List ${json_object} 622 623 ${id1}= Get From List ${list} 0 624 ${id1}= Convert To Integer ${id1} 625 ${id2}= Get From List ${list} 1 626 ${id2}= Convert To Integer ${id2} 627 628 Should Be True ${id1} > ${id2} 629 630 631Verify Total PEL Count 632 [Documentation] Verify total PEL count returned by peltool command. 633 [Tags] Verify_Total_PEL_Count 634 635 # Initially remove all logs. 636 Redfish Purge Event Log 637 638 # Generate a random number between 1-20. 639 ${random}= Evaluate random.randint(1, 20) modules=random 640 641 # Generate predictive error log multiple times. 642 FOR ${count} IN RANGE 0 ${random} 643 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 644 END 645 646 # Check PEL log count via peltool command and compare it with actual generated log count. 647 ${pel_records}= peltool -n 648 649 Should Be Equal ${pel_records['Number of PELs found']} ${random} 650 651 652Verify Listing Information Error 653 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 654 [Tags] Verify_Listing_Information_Error 655 656 # Initially remove all logs. 657 Redfish Purge Event Log 658 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 659 660 # Generate informational logs and verify that it would not get listed by peltool's list command. 661 ${pel_records}= peltool -l 662 ${ids}= Get Dictionary Keys ${pel_records} 663 Should Be Empty ${ids} 664 665 # Verify that information logs get listed using peltool's list command with -lfh option. 666 ${pel_records}= peltool -lfh 667 ${ids}= Get Dictionary Keys ${pel_records} 668 Should Not Be Empty ${ids} 669 ${id}= Get From List ${ids} 0 670 Should Contain ${pel_records['${id}']['Sev']} Informational 671 672 673Verify PEL Delete 674 [Documentation] Verify that peltool command can delete PEL log based on id. 675 [Tags] Verify_PEL_Delete 676 677 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 678 ${pel_ids}= Get PEL Log Via BMC CLI 679 ${id}= Get From List ${pel_ids} -1 680 Peltool -d ${id} False 681 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 682 683 684Verify Mandatory Fields For Predictive Error 685 [Documentation] Verify mandatory fields of predictive errors from pel information. 686 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 687 688 # Inject predictive error. 689 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 690 691 ${pel_ids}= Get PEL Log Via BMC CLI 692 ${pel_id}= Get From List ${pel_ids} -1 693 ${pel_output}= Peltool -i ${pel_id} 694 # Get all fields in predictive error log. 695 ${pel_sections}= Get Dictionary Keys ${pel_output} 696 697 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fields} 698 699 700Verify Mandatory Fields For Informational Error 701 [Documentation] Verify mandatory fields of informational error from pel information. 702 [Tags] Verify_Mandatory_Fields_For_Informational_Error 703 704 # Inject informational error. 705 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 706 ${pel_records}= Peltool -lfh 707 708 ${ids}= Get Dictionary Keys ${pel_records} 709 ${pel_id}= Get From List ${ids} -1 710 ${pel_output}= Peltool -i ${pel_id} 711 712 # Get all fields in the informational error log. 713 ${pel_sections}= Get Dictionary Keys ${pel_output} 714 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 715 ${contains}= Evaluate "${section}" in "${pel_sections}" 716 Should Be True ${contains} 717 END 718 719 720Verify PEL Log Not Offloaded To Host 721 [Documentation] Verify host transmission state of not offloaded PEL logs. 722 [Tags] Verify_PEL_Log_Not_Offloaded_To_Host 723 [Template] Verify PEL Transmission To Host 724 725 # error_type host_state expected_transmission_state 726 informational_error Off Not Sent 727 unrecoverable_error Off Not Sent 728 predictive_error Off Not Sent 729 730 731Verify PEL Log Offloaded To Host 732 [Documentation] Verify host transmission state of offloaded PEL logs. 733 [Tags] Verify_PEL_Log_Offloaded_To_Host 734 [Template] Verify PEL Transmission To Host 735 736 # error_type host_state expected_transmission_state 737 unrecoverable_error On Acked 738 predictive_error On Acked 739 informational_error On Acked 740 741 742*** Keywords *** 743 744Error Logging Rotation Policy 745 [Documentation] Verify that when maximum log limit is reached, given error logging type 746 ... are deleted when reached their max allocated space. 747 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 748 749 # Description of argument(s): 750 # error_log Error logs to be created (E.g. Informational BMC 3000 751 # stands for BMC created 3000 informational error logs). 752 # max_allocated_space_percentage The maximum percentage of disk usage for given error 753 # log type when maximum count/log size is reached. 754 # The maximum error log count is 3000. 755 756 Redfish.Login 757 758 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 759 # in BMC during the test. 760 Redfish Purge Event Log 761 762 @{lists}= Split String ${error_log_type} ,${SPACE} 763 764 ${length}= Get Length ${lists} 765 766 FOR ${list} IN RANGE ${length} 767 @{words}= Split String ${lists}[${list}] ${SPACE} 768 Create Error Log ${words}[0] ${words}[1] ${words}[2] 769 END 770 771 # Create an additional error log to exceed max error logs limit. 772 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 773 774 # Delay for BMC to perform delete older error logs when log limit exceeds. 775 Sleep 10s 776 777 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 778 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 779 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 780 781 ${disk_usage_percentage}= Get Disk Usage For Error Logs 782 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 783 ${percent_diff}= Evaluate abs(${percent_diff}) 784 785 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 786 ... Evaluate ${percent_diff} <= 0.5 787 ... ELSE 788 ... Set Variable True 789 790 # Check PEL log count via peltool command and compare it with actual generated log count. 791 ${pel_records}= peltool -n 792 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 793 # Number of logs can be 80% of the total logs created after trimming. 794 ${expected_max_record}= Evaluate 3000 * 0.8 795 796 Run Keyword If ${trimmed_as_expected} == False 797 ... Should Be True ${no_pel_records} <= ${expected_max_record} 798 799 800Create Error Log 801 [Documentation] Create an error log. 802 [Arguments] ${error_severity} ${error_creator} ${count} 803 804 # Description of argument(s): 805 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 806 # error_creator Name of error log's creator(E.g BMC, Host Boot) 807 # count Number of error logs to be generated. 808 809 FOR ${i} IN RANGE 0 ${count} 810 ${cmd}= Set Variable If 811 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 812 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 813 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 814 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 815 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 816 BMC Execute Command ${cmd} 817 END 818 819 820Get Disk Usage For Error Logs 821 [Documentation] Get disk usage percentage for error logs. 822 823 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 824 825 ${usage_output}= Fetch From Left ${usage_output} \/ 826 827 # Convert disk usage unit from KB to MB. 828 ${usage_output}= Evaluate ${usage_output} / 1024 829 830 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 831 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 832 833 [return] ${usage_percent} 834 835 836Get PEL Log IDs 837 [Documentation] Returns the list of PEL log IDs which contains given field's value. 838 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 839 840 # Description of argument(s): 841 # pel_section The section of PEL (e.g. Private Header, User Header). 842 # pel_field The PEL field (e.g. Event Severity, Event Type). 843 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 844 845 ${pel_ids}= Get PEL Log Via BMC CLI 846 @{pel_id_list}= Create List 847 848 FOR ${id} IN @{pel_ids} 849 ${pel_output}= Peltool -i ${id} 850 # Example of PEL output from "peltool -i <id>" command. 851 # [Private Header]: 852 # [Created at]: 08/24/1928 12:04:06 853 # [Created by]: 0x584D 854 # [Sub-section type]: 0 855 # [Entry Id]: 0x50000BB7 856 # [Platform Log Id]: 0x8200061D 857 # [CSSVER]: 858 # [Section Version]: 1 859 # [Creator Subsystem]: PHYP 860 # [BMC Event Log Id]: 341 861 # [Committed at]: 03/25/1920 12:06:22 862 # [User Header]: 863 # [Log Committed by]: 0x4552 864 # [Action Flags]: 865 # [0]: Report Externally 866 # [Subsystem]: I/O Subsystem 867 # [Event Type]: Miscellaneous, Informational Only 868 # [Sub-section type]: 0 869 # [Event Scope]: Entire Platform 870 # [Event Severity]: Informational Event 871 # [Host Transmission]: Not Sent 872 # [Section Version]: 1 873 874 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 875 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 876 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 877 END 878 Sort List ${pel_id_list} 879 880 [Return] ${pel_id_list} 881 882 883Get PEL Log Via BMC CLI 884 [Documentation] Returns the list of PEL IDs using BMC CLI. 885 886 ${pel_records}= Peltool -l 887 ${ids}= Get Dictionary Keys ${pel_records} 888 Sort List ${ids} 889 890 [Return] ${ids} 891 892 893Get PEL Field Value 894 [Documentation] Returns the value of given PEL's field. 895 [Arguments] ${pel_id} ${pel_section} ${pel_field} 896 897 # Description of argument(s): 898 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 899 # pel_section The section of PEL (e.g. Private Header, User Header) 900 # pel_field The PEL field (e.g. Event Severity, Event Type). 901 902 ${pel_output}= Peltool -i ${pel_id} 903 904 # Example of PEL output from "peltool -i <id>" command. 905 # [Private Header]: 906 # [Created at]: 08/24/1928 12:04:06 907 # [Created by]: 0x584D 908 # [Sub-section type]: 0 909 # [Entry Id]: 0x50000BB7 910 # [Platform Log Id]: 0x8200061D 911 # [CSSVER]: 912 # [Section Version]: 1 913 # [Creator Subsystem]: PHYP 914 # [BMC Event Log Id]: 341 915 # [Committed at]: 03/25/1920 12:06:22 916 # [User Header]: 917 # [Log Committed by]: 0x4552 918 # [Action Flags]: 919 # [0]: Report Externally 920 # [Subsystem]: I/O Subsystem 921 # [Event Type]: Miscellaneous, Informational Only 922 # [Sub-section type]: 0 923 # [Event Scope]: Entire Platform 924 # [Event Severity]: Informational Event 925 # [Host Transmission]: Not Sent 926 # [Section Version]: 1 927 928 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 929 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 930 931 [Return] ${pel_field_output} 932 933 934Verify PEL Transmission To Host 935 [Documentation] Inject PEL log of given type and verify its host transmission state. 936 [Arguments] ${error_type} ${host_state} ${expected_transmission_state} 937 938 # Description of argument(s): 939 # error_type Type of error log to be injected. 940 # host_state Host state which is required before error log injection. 941 # expected_transmission_state Expected host transmission state of PEL log. 942 943 # Get system in required state. 944 Run Keyword If '${host_state}' == 'Off' 945 ... Redfish Power Off stack_mode=skip 946 ... ELSE IF '${host_state}' == 'On' 947 ... Redfish Power On stack_mode=skip 948 949 Redfish Purge Event Log 950 951 # Inject required error log. 952 Run Keyword If "${error_type}" == "informational_error" 953 ... BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 954 ... ELSE IF "${error_type}" == "unrecoverable_error" 955 ... BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 956 ... ELSE IF "${error_type}" == "predictive_error" 957 ... BMC Execute Command ${CMD_PREDICTIVE_ERROR} 958 959 ${pel_records}= Get Pel Data From Bmc True True 960 ${ids}= Get Dictionary Keys ${pel_records} 961 ${pel_id}= Get From List ${ids} -1 962 963 # Check host transmission state for the cases where PEL is 964 # expected to be offloaded to HOST. 965 Run Keyword If "${expected_transmission_state}" == "Acked" 966 ... Wait Until Keyword Succeeds 2 min 10 sec 967 ... Check If PEL Transmission State Is Expected ${pel_id} Acked 968 969 # Adding delay before checking host transmission for the cases where PEL is 970 # not expected to be offloaded to HOST. 971 Run Keyword If "${expected_transmission_state}" == "Not sent" 972 ... Run Keywords Sleep 120s AND 973 ... Check If PEL Transmission State Is Expected ${pel_id} Not sent 974 975 976Check If PEL Transmission State Is Expected 977 [Documentation] Check if PEL's host transmission state is matching expected state. 978 [Arguments] ${pel_id} ${expected_transmission_state} 979 980 # Description of argument(s): 981 # expected_transmission_state Expected transmission state of PEL log. 982 983 # Check the transmission details in the error log. 984 ${host_transmission}= Get PEL Field Value ${pel_id} User Header Host Transmission 985 Should Be Equal ${host_transmission} ${expected_transmission_state} 986