1*** Settings *** 2Documentation This suite tests Platform Event Log (PEL) functionality of OpenBMC. 3 4Library ../../lib/pel_utils.py 5Variables ../../data/pel_variables.py 6Resource ../../lib/list_utils.robot 7Resource ../../lib/logging_utils.robot 8Resource ../../lib/connection_client.robot 9Resource ../../lib/openbmc_ffdc.robot 10 11Test Setup Redfish.Login 12Test Teardown Run Keywords Redfish.Logout AND FFDC On Test Case Fail 13 14 15*** Variables *** 16 17@{mandatory_pel_fields} Private Header User Header Primary SRC 18... Extended User Header Failing MTMS 19@{mandatory_Predictive_pel_fields} Private Header User Header Primary SRC 20... Extended User Header Failing MTMS User Data 21@{Mandatory_Informational_Pel_Fields} Private Header User Header Primary SRC 22... Extended User Header User Data 23 24*** Test Cases *** 25 26Create Test PEL Log And Verify 27 [Documentation] Create PEL log using busctl command and verify via peltool. 28 [Tags] Create_Test_PEL_Log_And_Verify 29 30 Redfish Purge Event Log 31 Create Test PEL Log 32 ${pel_id}= Get PEL Log Via BMC CLI 33 Should Not Be Empty ${pel_id} msg=System PEL log entry is empty. 34 35 36Verify PEL Log Details 37 [Documentation] Verify PEL log details via peltool. 38 [Tags] Verify_PEL_Log_Details 39 40 Redfish Purge Event Log 41 42 ${bmc_time1}= CLI Get BMC DateTime 43 Create Test PEL Log 44 ${bmc_time2}= CLI Get BMC DateTime 45 46 ${pel_records}= Peltool -l 47 48 # Example output from 'Peltool -l': 49 # pel_records: 50 # [0x50000012]: 51 # [CreatorID]: BMC 52 # [CompID]: 0x1000 53 # [PLID]: 0x50000012 54 # [Subsystem]: BMC Firmware 55 # [Message]: An application had an internal failure 56 # [SRC]: BD8D1002 57 # [Commit Time]: 03/02/2020 09:35:15 58 # [Sev]: Unrecoverable Error 59 60 ${ids}= Get Dictionary Keys ${pel_records} 61 ${id}= Get From List ${ids} 0 62 63 @{pel_fields}= Create List CreatorID Subsystem Message Sev 64 FOR ${field} IN @{pel_fields} 65 Valid Value pel_records['${id}']['${field}'] ['${PEL_DETAILS['${field}']}'] 66 END 67 68 Valid Value pel_records['${id}']['PLID'] ['${id}'] 69 70 # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value. 71 Should Match Regexp ${pel_records['${id}']['CompID']} [a-zA-Z0-9] 72 Should Match Regexp ${pel_records['${id}']['SRC']} [a-zA-Z0-9] 73 74 ${pel_date_time}= Convert Date ${pel_records['${id}']['Commit Time']} 75 ... date_format=%m/%d/%Y %H:%M:%S exclude_millis=yes 76 77 # Convert BMC and PEL time to epoch time before comparing. 78 ${bmc_time1_epoch}= Convert Date ${bmc_time1} epoch 79 ${pel_time_epoch}= Convert Date ${pel_date_time} epoch 80 ${bmc_time2_epoch}= Convert Date ${bmc_time2} epoch 81 82 Should Be True ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch} 83 84 85Verify Mandatory Sections Of Error Log PEL 86 [Documentation] Verify mandatory sections of error log PEL. 87 [Tags] Verify_Mandatory_Sections_Of_Error_Log_PEL 88 89 Create Test PEL Log 90 91 ${pel_ids}= Get PEL Log Via BMC CLI 92 ${pel_id}= Get From List ${pel_ids} -1 93 ${pel_output}= Peltool -i ${pel_id} 94 ${pel_sections}= Get Dictionary Keys ${pel_output} 95 96 List Should Contain Sub List ${pel_sections} ${mandatory_pel_fields} 97 98 99Verify PEL Log Persistence After BMC Reboot 100 [Documentation] Verify PEL log persistence after BMC reboot. 101 [Tags] Verify_PEL_Log_Persistence_After_BMC_Reboot 102 103 Create Test PEL Log 104 ${pel_before_reboot}= Get PEL Log Via BMC CLI 105 106 Redfish OBMC Reboot (off) 107 ${pel_after_reboot}= Get PEL Log Via BMC CLI 108 109 List Should Contain Sub List ${pel_after_reboot} ${pel_before_reboot} 110 111 112Verify PEL ID Numbering 113 [Documentation] Verify PEL ID numbering. 114 [Tags] Verify_PEL_ID_Numbering 115 116 Redfish Purge Event Log 117 Create Test PEL Log 118 Create Test PEL Log 119 120 ${pel_ids}= Get PEL Log Via BMC CLI 121 122 # Example of PEL IDs from PEL logs. 123 # [0x50000012]: <--- First PEL ID 124 # [CreatorID]: BMC 125 # [CompID]: 0x1000 126 # [PLID]: 0x50000012 127 # [Subsystem]: BMC Firmware 128 # [Message]: An application had an internal failure 129 # [SRC]: BD8D1002 130 # [Commit Time]: 03/02/2020 09:35:15 131 # [Sev]: Unrecoverable Error 132 # 133 # [0x50000013]: <--- Second PEL ID 134 # [CreatorID]: BMC 135 # [CompID]: 0x1000 136 # [PLID]: 0x50000013 137 # [Subsystem]: BMC Firmware 138 # [Message]: An application had an internal failure 139 # [SRC]: BD8D1002 140 # [Commit Time]: 03/02/2020 09:35:15 141 # [Sev]: Unrecoverable Error 142 143 Should Be True ${pel_ids[1]} == ${pel_ids[0]}+1 144 145Verify Machine Type Model And Serial Number 146 [Documentation] Verify machine type model and serial number from PEL. 147 [Tags] Verify_Machine_Type_Model_And_Serial_Number 148 149 Create Test PEL Log 150 151 ${pel_ids}= Get PEL Log Via BMC CLI 152 ${id}= Get From List ${pel_ids} -1 153 154 ${pel_serial_number}= Get PEL Field Value ${id} Failing MTMS Serial Number 155 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 156 ${pel_machine_type_model}= Get PEL Field Value ${id} Failing MTMS Machine Type Model 157 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 158 159 # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL. 160 # [Failing MTMS]: 161 # [Created by]: 0x2000 162 # [Machine Type Model]: 1234-ABC <---- Machine type 163 # [Section Version]: 1 164 # [Serial Number]: ABCDEFG <---- Serial number 165 # [Sub-section type]: 0 166 167 ${redfish_machine_model}= Redfish.Get Attribute /redfish/v1/Systems/system/ Model 168 ${redfish_machine_model}= Replace String Using Regexp ${redfish_machine_model} ^0+ ${EMPTY} 169 ${redfish_serial_number}= Redfish.Get Attribute /redfish/v1/Systems/system/ SerialNumber 170 ${redfish_serial_number}= Replace String Using Regexp ${redfish_serial_number} ^0+ ${EMPTY} 171 172 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 173 Valid Value pel_serial_number ['${redfish_serial_number}'] 174 175 # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL. 176 ${pel_machine_type_model}= Get PEL Field Value ${id} Extended User Header Reporting Machine Type 177 ${pel_machine_type_model}= Replace String Using Regexp ${pel_machine_type_model} ^0+ ${EMPTY} 178 ${pel_serial_number}= Get PEL Field Value ${id} Extended User Header Reporting Serial Number 179 ${pel_serial_number}= Replace String Using Regexp ${pel_serial_number} ^0+ ${EMPTY} 180 181 Valid Value pel_machine_type_model ['${redfish_machine_model}'] 182 Valid Value pel_serial_number ['${redfish_serial_number}'] 183 184 185Verify Host Off State From PEL 186 [Documentation] Verify Host off state from PEL. 187 [Tags] Verify_Host_Off_State_From_PEL 188 189 Redfish Power Off stack_mode=skip 190 Create Test PEL Log 191 192 ${pel_ids}= Get PEL Log Via BMC CLI 193 ${id}= Get From List ${pel_ids} -1 194 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 195 196 Valid Value pel_host_state ['Off'] 197 198 199Verify Host On State From PEL 200 [Documentation] Verify Host on state from PEL. 201 [Tags] Verify_Host_On_State_From_PEL 202 203 Redfish Power On stack_mode=skip 204 Redfish Purge Event Log 205 206 Create Test PEL Log 207 208 ${pel_ids}= Get PEL Log Via BMC CLI 209 ${id}= Get From List ${pel_ids} -1 210 ${pel_host_state}= Get PEL Field Value ${id} User Data HostState 211 212 Valid Value pel_host_state ['Running'] 213 214 215Verify BMC Version From PEL 216 [Documentation] Verify BMC Version from PEL. 217 [Tags] Verify_BMC_Version_From_PEL 218 219 Create Test PEL Log 220 221 ${pel_ids}= Get PEL Log Via BMC CLI 222 ${id}= Get From List ${pel_ids} -1 223 ${pel_bmc_version}= Get PEL Field Value ${id} User Data FW Version ID 224 225 ${bmc_version}= Get BMC Version 226 Valid Value bmc_version ['${bmc_version}'] 227 228 229Verify PEL Log After Host Poweron 230 [Documentation] Verify PEL log generation while booting host. 231 [Tags] Verify_PEL_Log_After_Host_Poweron 232 233 Redfish Power Off stack_mode=skip 234 Redfish Purge Event Log 235 Redfish Power On stack_mode=skip 236 237 ${pel_informational_error}= Get PEL Log IDs User Header Event Severity Informational Event 238 ${pel_bmc_created_error}= Get PEL Log IDs Private Header Creator Subsystem BMC 239 240 # Get BMC created non-infomational error. 241 ${pel_bmc_error}= Subtract Lists ${pel_bmc_created_error} ${pel_informational_error} 242 243 Should Be Empty ${pel_bmc_error} msg=Unexpected error log generated during Host poweron. 244 245 246Verify BMC Event Log ID 247 [Documentation] Verify BMC Event Log ID from PEL. 248 [Tags] Verify_BMC_Event_Log_ID 249 250 Redfish Purge Event Log 251 Create Test PEL Log 252 253 ${pel_ids}= Get PEL Log Via BMC CLI 254 ${pel_bmc_event_log_id}= Get PEL Field Value ${pel_ids[0]} Private Header BMC Event Log Id 255 256 # Example "BMC Event Log Id" field value from "Private Header" section of PEL. 257 # [Private Header]: 258 # [Created at]: 08/24/1928 12:04:06 259 # [Created by]: 0x584D 260 # [Sub-section type]: 0 261 # [Entry Id]: 0x50000BB7 262 # [Platform Log Id]: 0x8200061D 263 # [CSSVER]: 264 # [Section Version]: 1 265 # [Creator Subsystem]: PHYP 266 # [BMC Event Log Id]: 341 <---- BMC event log id value 267 # [Committed at]: 03/25/1920 12:06:22 268 269 ${redfish_event_logs}= Redfish.Get Properties /redfish/v1/Systems/system/LogServices/EventLog/Entries 270 271 # Example of redfish_event_logs output: 272 # redfish_event_logs: 273 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries 274 # [Name]: System Event Log Entries 275 # [Members@odata.count]: 1 276 # [@odata.type]: #LogEntryCollection.LogEntryCollection 277 # [Description]: Collection of System Event Log Entries 278 # [Members]: 279 # [0]: 280 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/235 281 # [Name]: System Event Log Entry 282 # [Severity]: Critical 283 # [EntryType]: Event 284 # [Created]: 2020-04-02T07:25:13+00:00 285 # [@odata.type]: #LogEntry.v1_5_1.LogEntry 286 # [Id]: 235 <----- Event log ID 287 # [Message]: xyz.openbmc_project.Common.Error.InternalFailure 288 289 Valid Value pel_bmc_event_log_id ['${redfish_event_logs['Members'][0]['Id']}'] 290 291 292Verify FRU Callout 293 [Documentation] Verify FRU callout entries from PEL log. 294 [Tags] Verify_FRU_Callout 295 296 # Power off the system to avoid unnecessary error logs during test. 297 Redfish Power Off stack_mode=skip 298 Create Test PEL Log FRU Callout 299 300 ${pel_ids}= Get PEL Log Via BMC CLI 301 ${id}= Get From List ${pel_ids} -1 302 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 303 304 # Example of PEL Callout Section from "peltool -i <id>" command. 305 # [Callouts]: 306 # [0]: 307 # [FRU Type]: Normal Hardware FRU 308 # [Priority]: Mandatory, replace all with this type as a unit 309 # [Location Code]: U78DA.ND1.1234567-P0 310 # [Part Number]: F191014 311 # [CCIN]: 2E2D 312 # [Serial Number]: YL2E2D010000 313 # [Callout Count]: 1 314 315 Valid Value pel_callout_section['Callout Count'] ['1'] 316 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Normal Hardware FRU'] 317 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 318 319 # Verify Location Code field of PEL callout with motherboard's Location Code. 320 ${busctl_output}= BMC Execute Command ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode 321 Should Be Equal ${pel_callout_section['Callouts'][0]['Location Code']} 322 ... ${busctl_output[0].split('"')[1].strip('"')} 323 324 # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output. 325 Should Match Regexp ${pel_callout_section['Callouts'][0]['CCIN']} [a-zA-Z0-9] 326 Should Match Regexp ${pel_callout_section['Callouts'][0]['Part Number']} [a-zA-Z0-9] 327 328 # Verify Serial Number field of PEL callout with motherboard's Serial Number. 329 ${busctl_output}= BMC Execute Command 330 ... ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber 331 Should Be Equal ${pel_callout_section['Callouts'][0]['Serial Number']} 332 ... ${busctl_output[0].split('"')[1].strip('"')} 333 334 335Verify Procedure And Symbolic FRU Callout 336 [Documentation] Verify procedure and symbolic FRU callout from PEL log. 337 [Tags] Verify_Procedure_And_Symbolic_FRU_Callout 338 339 Create Test PEL Log Procedure And Symbolic FRU Callout 340 341 ${pel_ids}= Get PEL Log Via BMC CLI 342 ${id}= Get From List ${pel_ids} -1 343 ${pel_callout_section}= Get PEL Field Value ${id} Primary SRC Callout Section 344 345 # Example of PEL Callout Section from "peltool -i <id>" command. 346 # [Callouts]: 347 # [0]: 348 # [Priority]: Mandatory, replace all with this type as a unit 349 # [Procedure Number]: BMCSP02 350 # [FRU Type]: Maintenance Procedure Required 351 # [1]: 352 # [Priority]: Medium Priority 353 # [Part Number]: SVCDOCS 354 # [FRU Type]: Symbolic FRU 355 # [Callout Count]: 2 356 357 Valid Value pel_callout_section['Callout Count'] ['2'] 358 359 # Verify procedural callout info. 360 361 Valid Value pel_callout_section['Callouts'][0]['FRU Type'] ['Maintenance Procedure Required'] 362 Should Contain ${pel_callout_section['Callouts'][0]['Priority']} Mandatory 363 # Verify if "Procedure Number" field of PEL has an alphanumeric value. 364 Should Match Regexp ${pel_callout_section['Callouts'][0]['Procedure']} [a-zA-Z0-9] 365 366 # Verify procedural callout info. 367 368 Valid Value pel_callout_section['Callouts'][1]['FRU Type'] ['Symbolic FRU'] 369 Should Contain ${pel_callout_section['Callouts'][1]['Priority']} Medium Priority 370 # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value. 371 Should Match Regexp ${pel_callout_section['Callouts'][1]['Part Number']} [a-zA-Z0-9] 372 373 374Verify PEL Log Entry For Event Log 375 [Documentation] Create an event log and verify PEL log entry in BMC for the same. 376 [Tags] Verify_PEL_Log_Entry_For_Event_Log 377 378 Redfish Purge Event Log 379 380 # Create an unrecoverable error log. 381 Create Test PEL Log Unrecoverable Error 382 383 ${elog_entry}= Get Event Logs 384 # Example of Redfish event logs: 385 # elog_entry: 386 # [0]: 387 # [Message]: BD802003 event in subsystem: Platform Firmware 388 # [Created]: 2020-04-20T01:55:22+00:00 389 # [Id]: 1 390 # [@odata.id]: /redfish/v1/Systems/system/LogServices/EventLog/Entries/1 391 # [@odata.type]: #LogEntry.v1_4_0.LogEntry 392 # [EntryType]: Event 393 # [Severity]: Critical 394 # [Name]: System Event Log Entry 395 396 ${redfish_log_time}= Convert Date ${elog_entry[0]["Created"]} epoch 397 398 ${pel_records}= Peltool -l 399 # Example output from 'Peltool -l': 400 # pel_records: 401 # [0x50000023]: 402 # [SRC]: BD802003 403 # [CreatorID]: BMC 404 # [Message]: This is a test error 405 # [CompID]: 0x2000 406 # [PLID]: 0x500053D9 407 # [Commit Time]: 04/20/2020 01:55:22 408 # [Subsystem]: Platform Firmware 409 # [Sev]: Unrecoverable Error 410 411 ${ids}= Get Dictionary Keys ${pel_records} 412 ${id}= Get From List ${ids} 0 413 ${pel_log_time}= Convert Date ${pel_records['${id}']['Commit Time']} epoch 414 ... date_format=%m/%d/%Y %H:%M:%S 415 416 # Verify that both Redfish event and PEL has log entry for internal error with same time stamp. 417 Should Contain Any ${pel_records['${id}']['Message']} test error ignore_case=True 418 Should Contain ${elog_entry[0]['Message']} 419 ... ${pel_records['${id}']['SRC']} ignore_case=True 420 421 Should Be Equal ${redfish_log_time} ${pel_log_time} 422 423 424Verify Delete All PEL 425 [Documentation] Verify deleting all PEL logs. 426 [Tags] Verify_Delete_All_PEL 427 428 Create Test PEL Log 429 Create Test PEL Log 430 Peltool --delete-all False 431 432 ${pel_ids}= Get PEL Log Via BMC CLI 433 Should Be Empty ${pel_ids} 434 435 436Verify Informational Error Log 437 [Documentation] Create an informational error log and verify. 438 [Tags] Verify_Informational_Error_Log 439 440 Redfish Purge Event Log 441 # Create an informational error log. 442 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 443 ${pel_records}= Peltool -lfh 444 445 # An example of information error log data: 446 # { 447 # "0x500006A0": { 448 # "SRC": "BD8D1002", 449 # "Message": "An application had an internal failure", 450 # "PLID": "0x500006A0", 451 # "CreatorID": "BMC", 452 # "Subsystem": "BMC Firmware", 453 # "Commit Time": "10/14/2020 11:41:38", 454 # "Sev": "Informational Event", 455 # "CompID": "0x1000" 456 # } 457 # } 458 459 ${ids}= Get Dictionary Keys ${pel_records} 460 ${id}= Get From List ${ids} 0 461 Should Contain ${pel_records['${id}']['Sev']} Informational 462 463 464Verify Predictable Error Log 465 [Documentation] Create a predictive error and verify. 466 [Tags] Verify_Predictable_Error_Log 467 468 # Create a predictable error log. 469 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 470 ${pel_records}= Peltool -l 471 472 # An example of predictive error log data: 473 # { 474 # "0x5000069E": { 475 # "SRC": "BD8D1002", 476 # "Message": "An application had an internal failure", 477 # "PLID": "0x5000069E", 478 # "CreatorID": "BMC", 479 # "Subsystem": "BMC Firmware", 480 # "Commit Time": "10/14/2020 11:40:07", 481 # "Sev": "Predictive Error", 482 # "CompID": "0x1000" 483 # } 484 # } 485 486 ${pel_ids}= Get PEL Log Via BMC CLI 487 ${id}= Get From List ${pel_ids} -1 488 Should Contain ${pel_records['${id}']['Sev']} Predictive 489 490 491Verify Unrecoverable Error Log 492 [Documentation] Create an unrecoverable error and verify. 493 [Tags] Verify_Unrecoverable_Error_Log 494 495 # Create an internal failure error log. 496 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 497 ${pel_records}= Peltool -l 498 499 # An example of unrecoverable error log data: 500 # { 501 # "0x50000CC5": { 502 # "SRC": "BD8D1002", 503 # "Message": "An application had an internal failure", 504 # "PLID": "0x50000CC5", 505 # "CreatorID": "BMC", 506 # "Subsystem": "BMC Firmware", 507 # "Commit Time": "04/01/2020 16:44:55", 508 # "Sev": "Unrecoverable Error", 509 # "CompID": "0x1000" 510 # } 511 # } 512 513 ${pel_ids}= Get PEL Log Via BMC CLI 514 ${id}= Get From List ${pel_ids} -1 515 Should Contain ${pel_records['${id}']['Sev']} Unrecoverable 516 517 518Verify Error Logging Rotation Policy 519 [Documentation] Verify error logging rotation policy. 520 [Tags] Verify_Error_Logging_Rotation_Policy 521 [Template] Error Logging Rotation Policy 522 523 # Error logs to be created % of total logging space when error 524 # log exceeds max limit. 525 Informational BMC 3000 15 526 Predictive BMC 3000 30 527 Unrecoverable BMC 3000 30 528 Informational BMC 1500, Predictive BMC 1500 45 529 Informational BMC 1500, Unrecoverable BMC 1500 45 530 Unrecoverable BMC 1500, Predictive BMC 1500 30 531 532 533Verify Error Logging Rotation Policy With All Types Of Errors 534 [Documentation] Verify error logging rotation policy with all types of errors. 535 [Tags] Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors 536 [Template] Error Logging Rotation Policy 537 538 # Error logs to be created % of total logging space when error 539 # log exceeds max limit. 540 Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000 45 541 542 543Verify Error Logging Rotation Policy With HOST Error Logs 544 [Documentation] Verify error logging rotation policy for non bmc error logs. 545 [Tags] Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs 546 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 547 ... /tmp/FILE_HOST_UNRECOVERABLE AND scp.Put File ${INFORMATIONAL_FILE_PATH} 548 ... /tmp/FILE_HOST_INFORMATIONAL 549 [Template] Error Logging Rotation Policy 550 551 # Error logs to be created % of total logging space when error 552 # log exceeds max limit. 553 Informational HOST 3000 15 554 Unrecoverable HOST 3000 30 555 Informational HOST 1500, Informational BMC 1500 30 556 Informational HOST 1500, Unrecoverable BMC 1500 45 557 Unrecoverable HOST 1500, Informational BMC 1500 45 558 Unrecoverable HOST 1500, Predictive BMC 1500 60 559 560 561Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs 562 [Documentation] Verify error logging rotation policy with unrecoverable HOST and BMC error logs. 563 [Tags] Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs 564 [Setup] Run Keywords Open Connection for SCP AND scp.Put File ${UNRECOVERABLE_FILE_PATH} 565 ... /tmp/FILE_NBMC_UNRECOVERABLE AND Redfish.Login 566 [Template] Error Logging Rotation Policy 567 568 # Error logs to be created % of total logging space when error 569 # log exceeds max limit. 570 Unrecoverable HOST 1500, Unrecoverable BMC 1500 60 571 572 573Verify Old Logs Are Deleted When Count Crosses Max 574 [Documentation] Verify that when the count crosses max, older logs are deleted. 575 [Tags] Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max 576 577 Redfish Purge Event Log 578 # Create 3000 error logs. 579 FOR ${count} IN RANGE ${3000} 580 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 581 END 582 583 # Retrieve the IDs of the logs. 584 ${pel_ids}= Get PEL Log Via BMC CLI 585 ${1st_id}= Get From List ${pel_ids} 0 586 ${3000th_id}= Get From List ${pel_ids} 2999 587 588 # Now create 3001st log to cross threshold limit and trigger error logs rotation. 589 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 590 591 # Wait few seconds for error logs rotation to complete. 592 Sleep 10s 593 594 # Now verify that first log is no more available but the 3000th is available. 595 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${1st_id} 596 Should Be True '${status}' == 'FAIL' 597 Should Contain ${output} PEL not found 598 599 ${status} ${output}= Run Keyword And Ignore Error Peltool -i ${3000th_id} 600 Should Be True '${status}' == 'PASS' 601 Should Not Contain ${output} PEL not found 602 603 604Verify Reverse Order Of PEL Logs 605 [Documentation] Verify PEL command to output PEL logs in reverse order. 606 [Tags] Verify_Reverse_Order_Of_PEL_Logs 607 608 Redfish Purge Event Log 609 610 # Below commands create unrecoverable error log at first and then the predictable error. 611 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 612 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 613 614 # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next. 615 ${pel_records}= peltool -lr 616 617 # It is found that, variables like dictionary always keep items in sorted order that makes 618 # this verification not possible, hence json is used to keep the items original order. 619 ${pel_records}= Convert To String ${pel_records} 620 ${json_string}= Replace String ${pel_records} ' " 621 ${json_object}= Evaluate json.loads('''${json_string}''') json 622 623 ${list}= Convert To List ${json_object} 624 625 ${id1}= Get From List ${list} 0 626 ${id1}= Convert To Integer ${id1} 627 ${id2}= Get From List ${list} 1 628 ${id2}= Convert To Integer ${id2} 629 630 Should Be True ${id1} > ${id2} 631 632 633Verify Total PEL Count 634 [Documentation] Verify total PEL count returned by peltool command. 635 [Tags] Verify_Total_PEL_Count 636 637 # Initially remove all logs. 638 Redfish Purge Event Log 639 640 # Generate a random number between 1-20. 641 ${random}= Evaluate random.randint(1, 20) modules=random 642 643 # Generate predictive error log multiple times. 644 FOR ${count} IN RANGE 0 ${random} 645 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 646 END 647 648 # Check PEL log count via peltool command and compare it with actual generated log count. 649 ${pel_records}= peltool -n 650 651 Should Be Equal ${pel_records['Number of PELs found']} ${random} 652 653 654Verify Listing Information Error 655 [Documentation] Verify that information error logs can only be listed using -lfh option of peltool. 656 [Tags] Verify_Listing_Information_Error 657 658 # Initially remove all logs. 659 Redfish Purge Event Log 660 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 661 662 # Generate informational logs and verify that it would not get listed by peltool's list command. 663 ${pel_records}= peltool -l 664 ${ids}= Get Dictionary Keys ${pel_records} 665 Should Be Empty ${ids} 666 667 # Verify that information logs get listed using peltool's list command with -lfh option. 668 ${pel_records}= peltool -lfh 669 ${ids}= Get Dictionary Keys ${pel_records} 670 Should Not Be Empty ${ids} 671 ${id}= Get From List ${ids} 0 672 Should Contain ${pel_records['${id}']['Sev']} Informational 673 674 675Verify PEL Delete 676 [Documentation] Verify that peltool command can delete PEL log based on id. 677 [Tags] Verify_PEL_Delete 678 679 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 680 ${pel_ids}= Get PEL Log Via BMC CLI 681 ${id}= Get From List ${pel_ids} -1 682 Peltool -d ${id} False 683 Run Keyword and Expect Error *PEL not found* Peltool -i ${id} 684 685 686Verify Mandatory Fields For Predictive Error 687 [Documentation] Verify mandatory fields of predictive errors from pel information. 688 [Tags] Verify_Mandatory_Fields_For_Predictive_Error 689 690 # Inject predictive error. 691 BMC Execute Command ${CMD_PREDICTIVE_ERROR} 692 693 ${pel_ids}= Get PEL Log Via BMC CLI 694 ${pel_id}= Get From List ${pel_ids} -1 695 ${pel_output}= Peltool -i ${pel_id} 696 # Get all fields in predictive error log. 697 ${pel_sections}= Get Dictionary Keys ${pel_output} 698 699 List Should Contain Sub List ${pel_sections} ${mandatory_Predictive_pel_fields} 700 701 702Verify Mandatory Fields For Informational Error 703 [Documentation] Verify mandatory fields of informational error from pel information. 704 [Tags] Verify_Mandatory_Fields_For_Informational_Error 705 706 # Inject informational error. 707 BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 708 ${pel_records}= Peltool -lfh 709 710 ${ids}= Get Dictionary Keys ${pel_records} 711 ${pel_id}= Get From List ${ids} -1 712 ${pel_output}= Peltool -i ${pel_id} 713 714 # Get all fields in the informational error log. 715 ${pel_sections}= Get Dictionary Keys ${pel_output} 716 FOR ${section} IN @{Mandatory_Informational_Pel_Fields} 717 ${contains}= Evaluate "${section}" in "${pel_sections}" 718 Should Be True ${contains} 719 END 720 721 722Verify PEL Log Not Offloaded To Host 723 [Documentation] Verify host transmission state of not offloaded PEL logs. 724 [Tags] Verify_PEL_Log_Not_Offloaded_To_Host 725 [Template] Verify PEL Transmission To Host 726 727 # error_type host_state expected_transmission_state 728 informational_error Off Not Sent 729 unrecoverable_error Off Not Sent 730 predictive_error Off Not Sent 731 732 733Verify PEL Log Offloaded To Host 734 [Documentation] Verify host transmission state of offloaded PEL logs. 735 [Tags] Verify_PEL_Log_Offloaded_To_Host 736 [Template] Verify PEL Transmission To Host 737 738 # error_type host_state expected_transmission_state 739 unrecoverable_error On Acked 740 predictive_error On Acked 741 informational_error On Acked 742 743 744*** Keywords *** 745 746Error Logging Rotation Policy 747 [Documentation] Verify that when maximum log limit is reached, given error logging type 748 ... are deleted when reached their max allocated space. 749 [Arguments] ${error_log_type} ${max_allocated_space_percentage} 750 751 # Description of argument(s): 752 # error_log Error logs to be created (E.g. Informational BMC 3000 753 # stands for BMC created 3000 informational error logs). 754 # max_allocated_space_percentage The maximum percentage of disk usage for given error 755 # log type when maximum count/log size is reached. 756 # The maximum error log count is 3000. 757 758 Redfish.Login 759 760 # Initially remove all logs. Purging is done to ensure that, only specific logs are present 761 # in BMC during the test. 762 Redfish Purge Event Log 763 764 @{lists}= Split String ${error_log_type} ,${SPACE} 765 766 ${length}= Get Length ${lists} 767 768 FOR ${list} IN RANGE ${length} 769 @{words}= Split String ${lists}[${list}] ${SPACE} 770 Create Error Log ${words}[0] ${words}[1] ${words}[2] 771 END 772 773 # Create an additional error log to exceed max error logs limit. 774 BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 775 776 # Delay for BMC to perform delete older error logs when log limit exceeds. 777 Sleep 10s 778 779 # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB 780 # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage 781 # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added. 782 783 ${disk_usage_percentage}= Get Disk Usage For Error Logs 784 ${percent_diff}= Evaluate ${disk_usage_percentage} - ${max_allocated_space_percentage} 785 ${percent_diff}= Evaluate abs(${percent_diff}) 786 787 ${trimmed_as_expected}= Run Keyword If ${disk_usage_percentage} > ${max_allocated_space_percentage} 788 ... Evaluate ${percent_diff} <= 0.5 789 ... ELSE 790 ... Set Variable True 791 792 # Check PEL log count via peltool command and compare it with actual generated log count. 793 ${pel_records}= peltool -n 794 ${no_pel_records}= Set Variable ${pel_records["Number of PELs found"]} 795 # Number of logs can be 80% of the total logs created after trimming. 796 ${expected_max_record}= Evaluate 3000 * 0.8 797 798 Run Keyword If ${trimmed_as_expected} == False 799 ... Should Be True ${no_pel_records} <= ${expected_max_record} 800 801 802Create Error Log 803 [Documentation] Create an error log. 804 [Arguments] ${error_severity} ${error_creator} ${count} 805 806 # Description of argument(s): 807 # error_severity Severity of the error (E.g. Informational, Unrecoberable or Predictive) 808 # error_creator Name of error log's creator(E.g BMC, Host Boot) 809 # count Number of error logs to be generated. 810 811 FOR ${i} IN RANGE 0 ${count} 812 ${cmd}= Set Variable If 813 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC' ${CMD_INFORMATIONAL_ERROR} 814 ... '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST' ${CMD_INFORMATIONAL_HOST_ERROR} 815 ... '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC' ${CMD_PREDICTIVE_ERROR} 816 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC' ${CMD_UNRECOVERABLE_ERROR} 817 ... '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST' ${CMD_UNRECOVERABLE_HOST_ERROR} 818 BMC Execute Command ${cmd} 819 END 820 821 822Get Disk Usage For Error Logs 823 [Documentation] Get disk usage percentage for error logs. 824 825 ${usage_output} ${stderr} ${rc}= BMC Execute Command du /var/lib/phosphor-logging/errors 826 827 ${usage_output}= Fetch From Left ${usage_output} \/ 828 829 # Convert disk usage unit from KB to MB. 830 ${usage_output}= Evaluate ${usage_output} / 1024 831 832 # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage. 833 ${usage_percent}= Evaluate ${usage_output} / 20 * 100 834 835 [return] ${usage_percent} 836 837 838Get PEL Log IDs 839 [Documentation] Returns the list of PEL log IDs which contains given field's value. 840 [Arguments] ${pel_section} ${pel_field} @{pel_field_value} 841 842 # Description of argument(s): 843 # pel_section The section of PEL (e.g. Private Header, User Header). 844 # pel_field The PEL field (e.g. Event Severity, Event Type). 845 # pel_field_value The list of PEL's field value (e.g. Unrecoverable Error). 846 847 ${pel_ids}= Get PEL Log Via BMC CLI 848 @{pel_id_list}= Create List 849 850 FOR ${id} IN @{pel_ids} 851 ${pel_output}= Peltool -i ${id} 852 # Example of PEL output from "peltool -i <id>" command. 853 # [Private Header]: 854 # [Created at]: 08/24/1928 12:04:06 855 # [Created by]: 0x584D 856 # [Sub-section type]: 0 857 # [Entry Id]: 0x50000BB7 858 # [Platform Log Id]: 0x8200061D 859 # [CSSVER]: 860 # [Section Version]: 1 861 # [Creator Subsystem]: PHYP 862 # [BMC Event Log Id]: 341 863 # [Committed at]: 03/25/1920 12:06:22 864 # [User Header]: 865 # [Log Committed by]: 0x4552 866 # [Action Flags]: 867 # [0]: Report Externally 868 # [Subsystem]: I/O Subsystem 869 # [Event Type]: Miscellaneous, Informational Only 870 # [Sub-section type]: 0 871 # [Event Scope]: Entire Platform 872 # [Event Severity]: Informational Event 873 # [Host Transmission]: Not Sent 874 # [Section Version]: 1 875 876 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 877 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 878 Run Keyword If '${pel_field_output}' in @{pel_field_value} Append To List ${pel_id_list} ${id} 879 END 880 Sort List ${pel_id_list} 881 882 [Return] ${pel_id_list} 883 884 885Get PEL Log Via BMC CLI 886 [Documentation] Returns the list of PEL IDs using BMC CLI. 887 888 ${pel_records}= Peltool -l 889 ${ids}= Get Dictionary Keys ${pel_records} 890 Sort List ${ids} 891 892 [Return] ${ids} 893 894 895Get PEL Field Value 896 [Documentation] Returns the value of given PEL's field. 897 [Arguments] ${pel_id} ${pel_section} ${pel_field} 898 899 # Description of argument(s): 900 # pel_id The ID of PEL (e.g. 0x5000002D, 0x5000002E). 901 # pel_section The section of PEL (e.g. Private Header, User Header) 902 # pel_field The PEL field (e.g. Event Severity, Event Type). 903 904 ${pel_output}= Peltool -i ${pel_id} 905 906 # Example of PEL output from "peltool -i <id>" command. 907 # [Private Header]: 908 # [Created at]: 08/24/1928 12:04:06 909 # [Created by]: 0x584D 910 # [Sub-section type]: 0 911 # [Entry Id]: 0x50000BB7 912 # [Platform Log Id]: 0x8200061D 913 # [CSSVER]: 914 # [Section Version]: 1 915 # [Creator Subsystem]: PHYP 916 # [BMC Event Log Id]: 341 917 # [Committed at]: 03/25/1920 12:06:22 918 # [User Header]: 919 # [Log Committed by]: 0x4552 920 # [Action Flags]: 921 # [0]: Report Externally 922 # [Subsystem]: I/O Subsystem 923 # [Event Type]: Miscellaneous, Informational Only 924 # [Sub-section type]: 0 925 # [Event Scope]: Entire Platform 926 # [Event Severity]: Informational Event 927 # [Host Transmission]: Not Sent 928 # [Section Version]: 1 929 930 ${pel_section_output}= Get From Dictionary ${pel_output} ${pel_section} 931 ${pel_field_output}= Get From Dictionary ${pel_section_output} ${pel_field} 932 933 [Return] ${pel_field_output} 934 935 936Verify PEL Transmission To Host 937 [Documentation] Inject PEL log of given type and verify its host transmission state. 938 [Arguments] ${error_type} ${host_state} ${expected_transmission_state} 939 940 # Description of argument(s): 941 # error_type Type of error log to be injected. 942 # host_state Host state which is required before error log injection. 943 # expected_transmission_state Expected host transmission state of PEL log. 944 945 # Get system in required state. 946 Run Keyword If '${host_state}' == 'Off' 947 ... Redfish Power Off stack_mode=skip 948 ... ELSE IF '${host_state}' == 'On' 949 ... Redfish Power On stack_mode=skip 950 951 Redfish Purge Event Log 952 953 # Inject required error log. 954 Run Keyword If "${error_type}" == "informational_error" 955 ... BMC Execute Command ${CMD_INFORMATIONAL_ERROR} 956 ... ELSE IF "${error_type}" == "unrecoverable_error" 957 ... BMC Execute Command ${CMD_UNRECOVERABLE_ERROR} 958 ... ELSE IF "${error_type}" == "predictive_error" 959 ... BMC Execute Command ${CMD_PREDICTIVE_ERROR} 960 961 ${pel_records}= Get Pel Data From Bmc True True 962 ${ids}= Get Dictionary Keys ${pel_records} 963 ${pel_id}= Get From List ${ids} -1 964 965 # Check host transmission state for the cases where PEL is 966 # expected to be offloaded to HOST. 967 Run Keyword If "${expected_transmission_state}" == "Acked" 968 ... Wait Until Keyword Succeeds 5 min 10 sec 969 ... Check If PEL Transmission State Is Expected ${pel_id} Acked 970 971 # Adding delay before checking host transmission for the cases where PEL is 972 # not expected to be offloaded to HOST. 973 Run Keyword If "${expected_transmission_state}" == "Not sent" 974 ... Run Keywords Sleep 120s AND 975 ... Check If PEL Transmission State Is Expected ${pel_id} Not sent 976 977 978Check If PEL Transmission State Is Expected 979 [Documentation] Check if PEL's host transmission state is matching expected state. 980 [Arguments] ${pel_id} ${expected_transmission_state} 981 982 # Description of argument(s): 983 # expected_transmission_state Expected transmission state of PEL log. 984 985 # Check the transmission details in the error log. 986 ${host_transmission}= Get PEL Field Value ${pel_id} User Header Host Transmission 987 Should Be Equal ${host_transmission} ${expected_transmission_state} 988