1*** Settings ***
2Documentation   This suite tests Platform Event Log (PEL) functionality of OpenBMC.
3
4Library         ../../lib/pel_utils.py
5Variables       ../../data/pel_variables.py
6Resource        ../../lib/list_utils.robot
7Resource        ../../lib/logging_utils.robot
8Resource        ../../lib/connection_client.robot
9Resource        ../../lib/openbmc_ffdc.robot
10
11Test Setup      Redfish.Login
12Test Teardown   Run Keywords  Redfish.Logout  AND  FFDC On Test Case Fail
13
14
15*** Variables ***
16
17@{mandatory_pel_fields}                Private Header  User Header  Primary SRC
18...                                    Extended User Header  Failing MTMS
19@{mandatory_Predictive_pel_fields}     Private Header  User Header  Primary SRC
20...                                    Extended User Header  Failing MTMS  User Data
21@{Mandatory_Informational_Pel_Fields}  Private Header  User Header  Primary SRC
22...                                    Extended User Header  User Data
23
24*** Test Cases ***
25
26Create Test PEL Log And Verify
27    [Documentation]  Create PEL log using busctl command and verify via peltool.
28    [Tags]  Create_Test_PEL_Log_And_Verify
29
30    Redfish Purge Event Log
31    Create Test PEL Log
32    ${pel_id}=  Get PEL Log Via BMC CLI
33    Should Not Be Empty  ${pel_id}  msg=System PEL log entry is empty.
34
35
36Verify PEL Log Details
37    [Documentation]  Verify PEL log details via peltool.
38    [Tags]  Verify_PEL_Log_Details
39
40    Redfish Purge Event Log
41
42    ${bmc_time1}=  CLI Get BMC DateTime
43    Create Test PEL Log
44    ${bmc_time2}=  CLI Get BMC DateTime
45
46    ${pel_records}=  Peltool  -l
47
48    # Example output from 'Peltool  -l':
49    # pel_records:
50    # [0x50000012]:
51    #   [CreatorID]:                  BMC
52    #   [CompID]:                     0x1000
53    #   [PLID]:                       0x50000012
54    #   [Subsystem]:                  BMC Firmware
55    #   [Message]:                    An application had an internal failure
56    #   [SRC]:                        BD8D1002
57    #   [Commit Time]:                03/02/2020  09:35:15
58    #   [Sev]:                        Unrecoverable Error
59
60    ${ids}=  Get Dictionary Keys  ${pel_records}
61    ${id}=  Get From List  ${ids}  0
62
63    @{pel_fields}=  Create List  CreatorID  Subsystem  Message  Sev
64    FOR  ${field}  IN  @{pel_fields}
65      Valid Value  pel_records['${id}']['${field}']  ['${PEL_DETAILS['${field}']}']
66    END
67
68    Valid Value  pel_records['${id}']['PLID']  ['${id}']
69
70    # Verify if "CompID" and "SRC" fields of PEL has alphanumeric value.
71    Should Match Regexp  ${pel_records['${id}']['CompID']}  [a-zA-Z0-9]
72    Should Match Regexp  ${pel_records['${id}']['SRC']}  [a-zA-Z0-9]
73
74    ${pel_date_time}=  Convert Date  ${pel_records['${id}']['Commit Time']}
75    ...  date_format=%m/%d/%Y %H:%M:%S  exclude_millis=yes
76
77    # Convert BMC and PEL time to epoch time before comparing.
78    ${bmc_time1_epoch}=  Convert Date  ${bmc_time1}  epoch
79    ${pel_time_epoch}=  Convert Date  ${pel_date_time}  epoch
80    ${bmc_time2_epoch}=  Convert Date  ${bmc_time2}  epoch
81
82    Should Be True  ${bmc_time1_epoch} <= ${pel_time_epoch} <= ${bmc_time2_epoch}
83
84
85Verify Mandatory Sections Of Error Log PEL
86    [Documentation]  Verify mandatory sections of error log PEL.
87    [Tags]  Verify_Mandatory_Sections_Of_Error_Log_PEL
88
89    Create Test PEL Log
90
91    ${pel_ids}=  Get PEL Log Via BMC CLI
92    ${pel_id}=  Get From List  ${pel_ids}  -1
93    ${pel_output}=  Peltool  -i ${pel_id}
94    ${pel_sections}=  Get Dictionary Keys  ${pel_output}
95
96    List Should Contain Sub List  ${pel_sections}  ${mandatory_pel_fields}
97
98
99Verify PEL Log Persistence After BMC Reboot
100    [Documentation]  Verify PEL log persistence after BMC reboot.
101    [Tags]  Verify_PEL_Log_Persistence_After_BMC_Reboot
102
103    Create Test PEL Log
104    ${pel_before_reboot}=  Get PEL Log Via BMC CLI
105
106    Redfish OBMC Reboot (off)
107    ${pel_after_reboot}=  Get PEL Log Via BMC CLI
108
109    List Should Contain Sub List  ${pel_after_reboot}  ${pel_before_reboot}
110
111
112Verify PEL ID Numbering
113    [Documentation]  Verify PEL ID numbering.
114    [Tags]  Verify_PEL_ID_Numbering
115
116    Redfish Purge Event Log
117    Create Test PEL Log
118    Create Test PEL Log
119
120    ${pel_ids}=  Get PEL Log Via BMC CLI
121
122    # Example of PEL IDs from PEL logs.
123    #  [0x50000012]:             <--- First PEL ID
124    #    [CreatorID]:                  BMC
125    #    [CompID]:                     0x1000
126    #    [PLID]:                       0x50000012
127    #    [Subsystem]:                  BMC Firmware
128    #    [Message]:                    An application had an internal failure
129    #    [SRC]:                        BD8D1002
130    #    [Commit Time]:                03/02/2020  09:35:15
131    #    [Sev]:                        Unrecoverable Error
132    #
133    #  [0x50000013]:             <--- Second PEL ID
134    #    [CreatorID]:                  BMC
135    #    [CompID]:                     0x1000
136    #    [PLID]:                       0x50000013
137    #    [Subsystem]:                  BMC Firmware
138    #    [Message]:                    An application had an internal failure
139    #    [SRC]:                        BD8D1002
140    #    [Commit Time]:                03/02/2020  09:35:15
141    #    [Sev]:                        Unrecoverable Error
142
143    Should Be True  ${pel_ids[1]} == ${pel_ids[0]}+1
144
145Verify Machine Type Model And Serial Number
146    [Documentation]  Verify machine type model and serial number from PEL.
147    [Tags]  Verify_Machine_Type_Model_And_Serial_Number
148
149    Create Test PEL Log
150
151    ${pel_ids}=  Get PEL Log Via BMC CLI
152    ${id}=  Get From List  ${pel_ids}  -1
153
154    ${pel_serial_number}=  Get PEL Field Value  ${id}  Failing MTMS  Serial Number
155    ${pel_serial_number}=  Replace String Using Regexp  ${pel_serial_number}  ^0+  ${EMPTY}
156    ${pel_machine_type_model}=  Get PEL Field Value  ${id}  Failing MTMS  Machine Type Model
157    ${pel_machine_type_model}=  Replace String Using Regexp  ${pel_machine_type_model}  ^0+  ${EMPTY}
158
159    # Example of "Machine Type Model" and "Serial Number" fields value from "Failing MTMS" section of PEL.
160    #  [Failing MTMS]:
161    #    [Created by]:                                 0x2000
162    #    [Machine Type Model]:                         1234-ABC   <---- Machine type
163    #    [Section Version]:                            1
164    #    [Serial Number]:                              ABCDEFG    <---- Serial number
165    #    [Sub-section type]:                           0
166
167    ${redfish_machine_model}=  Redfish.Get Attribute  /redfish/v1/Systems/system/  Model
168    ${redfish_machine_model}=  Replace String Using Regexp  ${redfish_machine_model}  ^0+  ${EMPTY}
169    ${redfish_serial_number}=  Redfish.Get Attribute  /redfish/v1/Systems/system/  SerialNumber
170    ${redfish_serial_number}=  Replace String Using Regexp  ${redfish_serial_number}  ^0+  ${EMPTY}
171
172    Valid Value  pel_machine_type_model  ['${redfish_machine_model}']
173    Valid Value  pel_serial_number  ['${redfish_serial_number}']
174
175    # Check "Machine Type Model" and "Serial Number" fields value from "Extended User Header" section of PEL.
176    ${pel_machine_type_model}=  Get PEL Field Value  ${id}  Extended User Header  Reporting Machine Type
177    ${pel_machine_type_model}=  Replace String Using Regexp  ${pel_machine_type_model}  ^0+  ${EMPTY}
178    ${pel_serial_number}=  Get PEL Field Value  ${id}  Extended User Header  Reporting Serial Number
179    ${pel_serial_number}=  Replace String Using Regexp  ${pel_serial_number}  ^0+  ${EMPTY}
180
181    Valid Value  pel_machine_type_model  ['${redfish_machine_model}']
182    Valid Value  pel_serial_number  ['${redfish_serial_number}']
183
184
185Verify Host Off State From PEL
186    [Documentation]  Verify Host off state from PEL.
187    [Tags]  Verify_Host_Off_State_From_PEL
188
189    Redfish Power Off  stack_mode=skip
190    Create Test PEL Log
191
192    ${pel_ids}=  Get PEL Log Via BMC CLI
193    ${id}=  Get From List  ${pel_ids}  -1
194    ${pel_host_state}=  Get PEL Field Value  ${id}  User Data  HostState
195
196    Valid Value  pel_host_state  ['Off']
197
198
199Verify Host On State From PEL
200    [Documentation]  Verify Host on state from PEL.
201    [Tags]  Verify_Host_On_State_From_PEL
202
203    Redfish Power On  stack_mode=skip
204    Redfish Purge Event Log
205
206    Create Test PEL Log
207
208    ${pel_ids}=  Get PEL Log Via BMC CLI
209    ${id}=  Get From List  ${pel_ids}  -1
210    ${pel_host_state}=  Get PEL Field Value  ${id}  User Data  HostState
211
212    Valid Value  pel_host_state  ['Running']
213
214
215Verify BMC Version From PEL
216    [Documentation]  Verify BMC Version from PEL.
217    [Tags]  Verify_BMC_Version_From_PEL
218
219    Create Test PEL Log
220
221    ${pel_ids}=  Get PEL Log Via BMC CLI
222    ${id}=  Get From List  ${pel_ids}  -1
223    ${pel_bmc_version}=  Get PEL Field Value  ${id}  User Data  FW Version ID
224
225    ${bmc_version}=  Get BMC Version
226    Valid Value  bmc_version  ['${bmc_version}']
227
228
229Verify PEL Log After Host Poweron
230    [Documentation]  Verify PEL log generation while booting host.
231    [Tags]  Verify_PEL_Log_After_Host_Poweron
232
233    Redfish Power Off  stack_mode=skip
234    Redfish Purge Event Log
235    Redfish Power On  stack_mode=skip
236
237    ${pel_informational_error}=  Get PEL Log IDs  User Header  Event Severity  Informational Event
238    ${pel_bmc_created_error}=  Get PEL Log IDs  Private Header  Creator Subsystem  BMC
239
240    # Get BMC created non-infomational error.
241    ${pel_bmc_error}=  Subtract Lists  ${pel_bmc_created_error}  ${pel_informational_error}
242
243    Should Be Empty  ${pel_bmc_error}  msg=Unexpected error log generated during Host poweron.
244
245
246Verify BMC Event Log ID
247    [Documentation]  Verify BMC Event Log ID from PEL.
248    [Tags]  Verify_BMC_Event_Log_ID
249
250    Redfish Purge Event Log
251    Create Test PEL Log
252
253    ${pel_ids}=  Get PEL Log Via BMC CLI
254    ${pel_bmc_event_log_id}=  Get PEL Field Value  ${pel_ids[0]}  Private Header  BMC Event Log Id
255
256    # Example "BMC Event Log Id" field value from "Private Header" section of PEL.
257    #  [Private Header]:
258    #    [Created at]:                 08/24/1928 12:04:06
259    #    [Created by]:                 0x584D
260    #    [Sub-section type]:           0
261    #    [Entry Id]:                   0x50000BB7
262    #    [Platform Log Id]:            0x8200061D
263    #    [CSSVER]:
264    #    [Section Version]:            1
265    #    [Creator Subsystem]:          PHYP
266    #    [BMC Event Log Id]:           341      <---- BMC event log id value
267    #    [Committed at]:               03/25/1920 12:06:22
268
269    ${redfish_event_logs}=  Redfish.Get Properties  /redfish/v1/Systems/system/LogServices/EventLog/Entries
270
271    # Example of redfish_event_logs output:
272    # redfish_event_logs:
273    #  [@odata.id]:                    /redfish/v1/Systems/system/LogServices/EventLog/Entries
274    #  [Name]:                         System Event Log Entries
275    #  [Members@odata.count]:          1
276    #  [@odata.type]:                  #LogEntryCollection.LogEntryCollection
277    #  [Description]:                  Collection of System Event Log Entries
278    #  [Members]:
279    #    [0]:
280    #      [@odata.id]:                /redfish/v1/Systems/system/LogServices/EventLog/Entries/235
281    #      [Name]:                     System Event Log Entry
282    #      [Severity]:                 Critical
283    #      [EntryType]:                Event
284    #      [Created]:                  2020-04-02T07:25:13+00:00
285    #      [@odata.type]:              #LogEntry.v1_5_1.LogEntry
286    #      [Id]:                       235          <----- Event log ID
287    #      [Message]:                  xyz.openbmc_project.Common.Error.InternalFailure
288
289    Valid Value  pel_bmc_event_log_id  ['${redfish_event_logs['Members'][0]['Id']}']
290
291
292Verify FRU Callout
293    [Documentation]  Verify FRU callout entries from PEL log.
294    [Tags]  Verify_FRU_Callout
295
296    # Power off the system to avoid unnecessary error logs during test.
297    Redfish Power Off  stack_mode=skip
298    Create Test PEL Log  FRU Callout
299
300    ${pel_ids}=  Get PEL Log Via BMC CLI
301    ${id}=  Get From List  ${pel_ids}  -1
302    ${pel_callout_section}=  Get PEL Field Value  ${id}  Primary SRC  Callout Section
303
304    # Example of PEL Callout Section from "peltool -i <id>" command.
305    #  [Callouts]:
306    #    [0]:
307    #      [FRU Type]:                 Normal Hardware FRU
308    #      [Priority]:                 Mandatory, replace all with this type as a unit
309    #      [Location Code]:            U78DA.ND1.1234567-P0
310    #      [Part Number]:              F191014
311    #      [CCIN]:                     2E2D
312    #      [Serial Number]:            YL2E2D010000
313    #  [Callout Count]:                1
314
315    Valid Value  pel_callout_section['Callout Count']  ['1']
316    Valid Value  pel_callout_section['Callouts'][0]['FRU Type']  ['Normal Hardware FRU']
317    Should Contain  ${pel_callout_section['Callouts'][0]['Priority']}  Mandatory
318
319    # Verify Location Code field of PEL callout with motherboard's Location Code.
320    ${busctl_output}=  BMC Execute Command  ${CMD_INVENTORY_PREFIX} com.ibm.ipzvpd.Location LocationCode
321    Should Be Equal  ${pel_callout_section['Callouts'][0]['Location Code']}
322    ...  ${busctl_output[0].split('"')[1].strip('"')}
323
324    # TODO: Compare CCIN and part number fields of PEL callout with Redfish or busctl output.
325    Should Match Regexp  ${pel_callout_section['Callouts'][0]['CCIN']}  [a-zA-Z0-9]
326    Should Match Regexp  ${pel_callout_section['Callouts'][0]['Part Number']}  [a-zA-Z0-9]
327
328    # Verify Serial Number field of PEL callout with motherboard's Serial Number.
329    ${busctl_output}=  BMC Execute Command
330    ...  ${CMD_INVENTORY_PREFIX} xyz.openbmc_project.Inventory.Decorator.Asset SerialNumber
331    Should Be Equal  ${pel_callout_section['Callouts'][0]['Serial Number']}
332    ...  ${busctl_output[0].split('"')[1].strip('"')}
333
334
335Verify Procedure And Symbolic FRU Callout
336    [Documentation]  Verify procedure and symbolic FRU callout from PEL log.
337    [Tags]  Verify_Procedure_And_Symbolic_FRU_Callout
338
339    Create Test PEL Log   Procedure And Symbolic FRU Callout
340
341    ${pel_ids}=  Get PEL Log Via BMC CLI
342    ${id}=  Get From List  ${pel_ids}  -1
343    ${pel_callout_section}=  Get PEL Field Value  ${id}  Primary SRC  Callout Section
344
345    # Example of PEL Callout Section from "peltool -i <id>" command.
346    #  [Callouts]:
347    #    [0]:
348    #      [Priority]:                                 Mandatory, replace all with this type as a unit
349    #      [Procedure Number]:                         BMCSP02
350    #      [FRU Type]:                                 Maintenance Procedure Required
351    #    [1]:
352    #      [Priority]:                                 Medium Priority
353    #      [Part Number]:                              SVCDOCS
354    #      [FRU Type]:                                 Symbolic FRU
355    #  [Callout Count]:                                2
356
357    Valid Value  pel_callout_section['Callout Count']  ['2']
358
359    # Verify procedural callout info.
360
361    Valid Value  pel_callout_section['Callouts'][0]['FRU Type']  ['Maintenance Procedure Required']
362    Should Contain  ${pel_callout_section['Callouts'][0]['Priority']}  Mandatory
363    # Verify if "Procedure Number" field of PEL has an alphanumeric value.
364    Should Match Regexp  ${pel_callout_section['Callouts'][0]['Procedure']}  [a-zA-Z0-9]
365
366    # Verify procedural callout info.
367
368    Valid Value  pel_callout_section['Callouts'][1]['FRU Type']  ['Symbolic FRU']
369    Should Contain  ${pel_callout_section['Callouts'][1]['Priority']}  Medium Priority
370    # Verify if "Part Number" field of Symbolic FRU has an alphanumeric value.
371    Should Match Regexp  ${pel_callout_section['Callouts'][1]['Part Number']}  [a-zA-Z0-9]
372
373
374Verify PEL Log Entry For Event Log
375    [Documentation]  Create an event log and verify PEL log entry in BMC for the same.
376    [Tags]  Verify_PEL_Log_Entry_For_Event_Log
377
378    Redfish Purge Event Log
379
380    # Create an unrecoverable error log.
381    Create Test PEL Log  Unrecoverable Error
382
383    ${elog_entry}=  Get Event Logs
384    # Example of Redfish event logs:
385    # elog_entry:
386    #  [0]:
387    #    [Message]:                             BD802003 event in subsystem: Platform Firmware
388    #    [Created]:                             2020-04-20T01:55:22+00:00
389    #    [Id]:                                  1
390    #    [@odata.id]:                           /redfish/v1/Systems/system/LogServices/EventLog/Entries/1
391    #    [@odata.type]:                         #LogEntry.v1_4_0.LogEntry
392    #    [EntryType]:                           Event
393    #    [Severity]:                            Critical
394    #    [Name]:                                System Event Log Entry
395
396    ${redfish_log_time}=  Convert Date  ${elog_entry[0]["Created"]}  epoch
397
398    ${pel_records}=  Peltool  -l
399    # Example output from 'Peltool  -l':
400    # pel_records:
401    # [0x50000023]:
402    #   [SRC]:                                   BD802003
403    #   [CreatorID]:                             BMC
404    #   [Message]:                               This is a test error
405    #   [CompID]:                                0x2000
406    #   [PLID]:                                  0x500053D9
407    #   [Commit Time]:                           04/20/2020 01:55:22
408    #   [Subsystem]:                             Platform Firmware
409    #   [Sev]:                                   Unrecoverable Error
410
411    ${ids}=  Get Dictionary Keys  ${pel_records}
412    ${id}=  Get From List  ${ids}  0
413    ${pel_log_time}=  Convert Date  ${pel_records['${id}']['Commit Time']}  epoch
414    ...  date_format=%m/%d/%Y %H:%M:%S
415
416    # Verify that both Redfish event and PEL has log entry for internal error with same time stamp.
417    Should Contain Any  ${pel_records['${id}']['Message']}  test error  ignore_case=True
418    Should Contain  ${elog_entry[0]['Message']}
419    ...  ${pel_records['${id}']['SRC']}  ignore_case=True
420
421    Should Be Equal  ${redfish_log_time}  ${pel_log_time}
422
423
424Verify Delete All PEL
425    [Documentation]  Verify deleting all PEL logs.
426    [Tags]  Verify_Delete_All_PEL
427
428    Create Test PEL Log
429    Create Test PEL Log
430    Peltool  --delete-all  False
431
432    ${pel_ids}=  Get PEL Log Via BMC CLI
433    Should Be Empty  ${pel_ids}
434
435
436Verify Informational Error Log
437    [Documentation]  Create an informational error log and verify.
438    [Tags]  Verify_Informational_Error_Log
439
440    Redfish Purge Event Log
441    # Create an informational error log.
442    BMC Execute Command  ${CMD_INFORMATIONAL_ERROR}
443    ${pel_records}=  Peltool  -lfh
444
445    # An example of information error log data:
446    # {
447    #    "0x500006A0": {
448    #            "SRC": "BD8D1002",
449    #            "Message": "An application had an internal failure",
450    #            "PLID": "0x500006A0",
451    #            "CreatorID": "BMC",
452    #            "Subsystem": "BMC Firmware",
453    #            "Commit Time": "10/14/2020 11:41:38",
454    #            "Sev": "Informational Event",
455    #            "CompID": "0x1000"
456    #    }
457    # }
458
459    ${ids}=  Get Dictionary Keys  ${pel_records}
460    ${id}=  Get From List  ${ids}  0
461    Should Contain  ${pel_records['${id}']['Sev']}  Informational
462
463
464Verify Predictable Error Log
465    [Documentation]  Create a predictive error and verify.
466    [Tags]  Verify_Predictable_Error_Log
467
468    # Create a predictable error log.
469    BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
470    ${pel_records}=  Peltool  -l
471
472    # An example of predictive error log data:
473    # {
474    #    "0x5000069E": {
475    #            "SRC": "BD8D1002",
476    #            "Message": "An application had an internal failure",
477    #            "PLID": "0x5000069E",
478    #            "CreatorID": "BMC",
479    #            "Subsystem": "BMC Firmware",
480    #            "Commit Time": "10/14/2020 11:40:07",
481    #            "Sev": "Predictive Error",
482    #            "CompID": "0x1000"
483    #    }
484    # }
485
486    ${pel_ids}=  Get PEL Log Via BMC CLI
487    ${id}=  Get From List  ${pel_ids}  -1
488    Should Contain  ${pel_records['${id}']['Sev']}  Predictive
489
490
491Verify Unrecoverable Error Log
492    [Documentation]  Create an unrecoverable error and verify.
493    [Tags]  Verify_Unrecoverable_Error_Log
494
495    # Create an internal failure error log.
496    BMC Execute Command  ${CMD_UNRECOVERABLE_ERROR}
497    ${pel_records}=  Peltool  -l
498
499    # An example of unrecoverable error log data:
500    # {
501    #    "0x50000CC5": {
502    #            "SRC": "BD8D1002",
503    #            "Message": "An application had an internal failure",
504    #            "PLID": "0x50000CC5",
505    #            "CreatorID": "BMC",
506    #            "Subsystem": "BMC Firmware",
507    #            "Commit Time": "04/01/2020 16:44:55",
508    #            "Sev": "Unrecoverable Error",
509    #            "CompID": "0x1000"
510    #    }
511    # }
512
513    ${pel_ids}=  Get PEL Log Via BMC CLI
514    ${id}=  Get From List  ${pel_ids}  -1
515    Should Contain  ${pel_records['${id}']['Sev']}  Unrecoverable
516
517
518Verify Error Logging Rotation Policy
519    [Documentation]  Verify error logging rotation policy.
520    [Tags]  Verify_Error_Logging_Rotation_Policy
521    [Template]  Error Logging Rotation Policy
522
523    # Error logs to be created                                % of total logging space when error
524    #                                                         log exceeds max limit.
525    Informational BMC 3000                                                       15
526    Predictive BMC 3000                                                          30
527    Unrecoverable BMC 3000                                                       30
528    Informational BMC 1500, Predictive BMC 1500                                  45
529    Informational BMC 1500, Unrecoverable BMC 1500                               45
530    Unrecoverable BMC 1500, Predictive BMC 1500                                  30
531
532
533Verify Error Logging Rotation Policy With All Types Of Errors
534    [Documentation]  Verify error logging rotation policy with all types of errors.
535    [Tags]  Verify_Error_Logging_Rotation_Policy_With_All_Types_Of_Errors
536    [Template]  Error Logging Rotation Policy
537
538    # Error logs to be created                                           % of total logging space when error
539    #                                                                    log exceeds max limit.
540    Unrecoverable BMC 1000, Informational BMC 1000, Predictive BMC 1000          45
541
542
543Verify Error Logging Rotation Policy With HOST Error Logs
544    [Documentation]  Verify error logging rotation policy for non bmc error logs.
545    [Tags]  Verify_Error_Logging_Rotation_Policy_With_HOST_Error_Logs
546    [Setup]  Run Keywords  Open Connection for SCP  AND  scp.Put File  ${UNRECOVERABLE_FILE_PATH}
547    ...  /tmp/FILE_HOST_UNRECOVERABLE  AND  scp.Put File  ${INFORMATIONAL_FILE_PATH}
548    ...  /tmp/FILE_HOST_INFORMATIONAL
549    [Template]  Error Logging Rotation Policy
550
551    # Error logs to be created                                % of total logging space when error
552    #                                                         log exceeds max limit.
553    Informational HOST 3000                                                   15
554    Unrecoverable HOST 3000                                                   30
555    Informational HOST 1500, Informational BMC 1500                           30
556    Informational HOST 1500, Unrecoverable BMC 1500                           45
557    Unrecoverable HOST 1500, Informational BMC 1500                           45
558    Unrecoverable HOST 1500, Predictive BMC 1500                              60
559
560
561Verify Error Logging Rotation Policy With Unrecoverable HOST And BMC Error Logs
562    [Documentation]  Verify error logging rotation policy with unrecoverable HOST and BMC error logs.
563    [Tags]  Verify_Error_Logging_Rotation_Policy_With_Unrecoverable_HOST_And_BMC_Error_Logs
564    [Setup]  Run Keywords  Open Connection for SCP  AND  scp.Put File  ${UNRECOVERABLE_FILE_PATH}
565    ...  /tmp/FILE_NBMC_UNRECOVERABLE  AND  Redfish.Login
566    [Template]  Error Logging Rotation Policy
567
568    # Error logs to be created                                % of total logging space when error
569    #                                                         log exceeds max limit.
570    Unrecoverable HOST 1500, Unrecoverable BMC 1500                           60
571
572
573Verify Old Logs Are Deleted When Count Crosses Max
574    [Documentation]  Verify that when the count crosses max, older logs are deleted.
575    [Tags]  Verify_Old_Logs_Are_Deleted_When_Count_Crosses_Max
576
577    Redfish Purge Event Log
578    # Create 3000 error logs.
579    FOR  ${count}  IN RANGE  ${3000}
580        BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
581    END
582
583    # Retrieve the IDs of the logs.
584    ${pel_ids}=  Get PEL Log Via BMC CLI
585    ${1st_id}=  Get From List  ${pel_ids}  0
586    ${3000th_id}=  Get From List  ${pel_ids}  2999
587
588    # Now create 3001st log to cross threshold limit and trigger error logs rotation.
589    BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
590
591    # Wait few seconds for error logs rotation to complete.
592    Sleep  10s
593
594    # Now verify that first log is no more available but the 3000th is available.
595    ${status}  ${output}=  Run Keyword And Ignore Error  Peltool  -i ${1st_id}
596    Should Be True  '${status}' == 'FAIL'
597    Should Contain  ${output}  PEL not found
598
599    ${status}  ${output}=  Run Keyword And Ignore Error  Peltool  -i ${3000th_id}
600    Should Be True  '${status}' == 'PASS'
601    Should Not Contain  ${output}  PEL not found
602
603
604Verify Reverse Order Of PEL Logs
605    [Documentation]  Verify PEL command to output PEL logs in reverse order.
606    [Tags]  Verify_Reverse_Order_Of_PEL_Logs
607
608    Redfish Purge Event Log
609
610    # Below commands create unrecoverable error log at first and then the predictable error.
611    BMC Execute Command  ${CMD_UNRECOVERABLE_ERROR}
612    BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
613
614    # Using peltool -lr, recent PELs appear first. Hence the ID of first PEL is greater than the next.
615    ${pel_records}=  peltool  -lr
616
617    # It is found that, variables like dictionary always keep items in sorted order that makes
618    # this verification not possible, hence json is used to keep the items original order.
619    ${pel_records}=  Convert To String  ${pel_records}
620    ${json_string}=  Replace String  ${pel_records}  '  "
621    ${json_object}=  Evaluate  json.loads('''${json_string}''')  json
622
623    ${list}=  Convert To List  ${json_object}
624
625    ${id1}=  Get From List   ${list}  0
626    ${id1}=  Convert To Integer  ${id1}
627    ${id2}=  Get From List   ${list}  1
628    ${id2}=  Convert To Integer  ${id2}
629
630    Should Be True  ${id1} > ${id2}
631
632
633Verify Total PEL Count
634    [Documentation]  Verify total PEL count returned by peltool command.
635    [Tags]  Verify_Total_PEL_Count
636
637    # Initially remove all logs.
638    Redfish Purge Event Log
639
640    # Generate a random number between 1-20.
641    ${random}=  Evaluate  random.randint(1, 20)  modules=random
642
643    # Generate predictive error log multiple times.
644    FOR  ${count}  IN RANGE  0  ${random}
645      BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
646    END
647
648    # Check PEL log count via peltool command and compare it with actual generated log count.
649    ${pel_records}=  peltool  -n
650
651    Should Be Equal  ${pel_records['Number of PELs found']}   ${random}
652
653
654Verify Listing Information Error
655    [Documentation]  Verify that information error logs can only be listed using -lfh option of peltool.
656    [Tags]  Verify_Listing_Information_Error
657
658    # Initially remove all logs.
659    Redfish Purge Event Log
660    BMC Execute Command  ${CMD_INFORMATIONAL_ERROR}
661
662    # Generate informational logs and verify that it would not get listed by peltool's list command.
663    ${pel_records}=  peltool  -l
664    ${ids}=  Get Dictionary Keys  ${pel_records}
665    Should Be Empty  ${ids}
666
667    # Verify that information logs get listed using peltool's list command with -lfh option.
668    ${pel_records}=  peltool  -lfh
669    ${ids}=  Get Dictionary Keys  ${pel_records}
670    Should Not Be Empty  ${ids}
671    ${id}=  Get From List  ${ids}  0
672    Should Contain  ${pel_records['${id}']['Sev']}  Informational
673
674
675Verify PEL Delete
676    [Documentation]  Verify that peltool command can delete PEL log based on id.
677    [Tags]  Verify_PEL_Delete
678
679    BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
680    ${pel_ids}=  Get PEL Log Via BMC CLI
681    ${id}=  Get From List  ${pel_ids}  -1
682    Peltool  -d ${id}  False
683    Run Keyword and Expect Error  *PEL not found*  Peltool  -i ${id}
684
685
686Verify Mandatory Fields For Predictive Error
687    [Documentation]  Verify mandatory fields of predictive errors from pel information.
688    [Tags]  Verify_Mandatory_Fields_For_Predictive_Error
689
690    # Inject predictive error.
691    BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
692
693    ${pel_ids}=  Get PEL Log Via BMC CLI
694    ${pel_id}=  Get From List  ${pel_ids}  -1
695    ${pel_output}=  Peltool  -i ${pel_id}
696    # Get all fields in predictive error log.
697    ${pel_sections}=  Get Dictionary Keys  ${pel_output}
698
699    List Should Contain Sub List  ${pel_sections}  ${mandatory_Predictive_pel_fields}
700
701
702Verify Mandatory Fields For Informational Error
703    [Documentation]  Verify mandatory fields of informational error from pel information.
704    [Tags]  Verify_Mandatory_Fields_For_Informational_Error
705
706    # Inject informational error.
707    BMC Execute Command  ${CMD_INFORMATIONAL_ERROR}
708    ${pel_records}=  Peltool  -lfh
709
710    ${ids}=  Get Dictionary Keys  ${pel_records}
711    ${pel_id}=  Get From List  ${ids}  -1
712    ${pel_output}=  Peltool  -i ${pel_id}
713
714    # Get all fields in the informational error log.
715    ${pel_sections}=  Get Dictionary Keys  ${pel_output}
716    FOR  ${section}  IN  @{Mandatory_Informational_Pel_Fields}
717        ${contains}=  Evaluate  "${section}" in "${pel_sections}"
718        Should Be True  ${contains}
719    END
720
721
722Verify PEL Log Not Offloaded To Host
723    [Documentation]  Verify host transmission state of not offloaded PEL logs.
724    [Tags]  Verify_PEL_Log_Not_Offloaded_To_Host
725    [Template]  Verify PEL Transmission To Host
726
727    # error_type            host_state      expected_transmission_state
728      informational_error   Off             Not Sent
729      unrecoverable_error   Off             Not Sent
730      predictive_error      Off             Not Sent
731
732
733Verify PEL Log Offloaded To Host
734    [Documentation]  Verify host transmission state of offloaded PEL logs.
735    [Tags]  Verify_PEL_Log_Offloaded_To_Host
736    [Template]  Verify PEL Transmission To Host
737
738    # error_type            host_state      expected_transmission_state
739      unrecoverable_error   On              Acked
740      predictive_error      On              Acked
741      informational_error   On              Acked
742
743
744*** Keywords ***
745
746Error Logging Rotation Policy
747    [Documentation]  Verify that when maximum log limit is reached, given error logging type
748    ...  are deleted when reached their max allocated space.
749    [Arguments]  ${error_log_type}  ${max_allocated_space_percentage}
750
751    # Description of argument(s):
752    # error_log                           Error logs to be created (E.g. Informational BMC 3000
753    #                                     stands for BMC created 3000 informational error logs).
754    # max_allocated_space_percentage      The maximum percentage of disk usage for given error
755    #                                     log type when maximum count/log size is reached.
756    #                                     The maximum error log count is 3000.
757
758    Redfish.Login
759
760    # Initially remove all logs. Purging is done to ensure that, only specific logs are present
761    # in BMC during the test.
762    Redfish Purge Event Log
763
764    @{lists}=  Split String  ${error_log_type}  ,${SPACE}
765
766    ${length}=  Get Length  ${lists}
767
768    FOR  ${list}  IN RANGE  ${length}
769        @{words}=  Split String  ${lists}[${list}]  ${SPACE}
770        Create Error Log  ${words}[0]  ${words}[1]  ${words}[2]
771    END
772
773    # Create an additional error log to exceed max error logs limit.
774    BMC Execute Command  ${CMD_UNRECOVERABLE_ERROR}
775
776    # Delay for BMC to perform delete older error logs when log limit exceeds.
777    Sleep  10s
778
779    # Verify disk usage is around max allocated space. Maximum usage is around 3MB not exactly 3MB
780    # (for informational log) and around 6 MB for unrecoverable / predictive error log. So, usage
781    # percentage is NOT exactly 15% and 30%. So, an error/accuracy factor 0.5 percent is added.
782
783    ${disk_usage_percentage}=  Get Disk Usage For Error Logs
784    ${percent_diff}=  Evaluate  ${disk_usage_percentage} - ${max_allocated_space_percentage}
785    ${percent_diff}=   Evaluate  abs(${percent_diff})
786
787    ${trimmed_as_expected}=  Run Keyword If  ${disk_usage_percentage} > ${max_allocated_space_percentage}
788    ...  Evaluate  ${percent_diff} <= 0.5
789    ...  ELSE
790    ...  Set Variable  True
791
792    # Check PEL log count via peltool command and compare it with actual generated log count.
793    ${pel_records}=  peltool  -n
794    ${no_pel_records}=  Set Variable  ${pel_records["Number of PELs found"]}
795    # Number of logs can be 80% of the total logs created after trimming.
796    ${expected_max_record}=   Evaluate  3000 * 0.8
797
798    Run Keyword If  ${trimmed_as_expected} == False
799    ...  Should Be True  ${no_pel_records} <= ${expected_max_record}
800
801
802Create Error Log
803    [Documentation]  Create an error log.
804    [Arguments]  ${error_severity}   ${error_creator}   ${count}
805
806    # Description of argument(s):
807    # error_severity             Severity of the error (E.g. Informational, Unrecoberable or Predictive)
808    # error_creator              Name of error log's creator(E.g BMC, Host Boot)
809    # count                      Number of error logs to be generated.
810
811    FOR  ${i}  IN RANGE  0  ${count}
812        ${cmd}=  Set Variable If
813        ...  '${error_severity}' == 'Informational' and '${error_creator}' == 'BMC'  ${CMD_INFORMATIONAL_ERROR}
814        ...  '${error_severity}' == 'Informational' and '${error_creator}' == 'HOST'  ${CMD_INFORMATIONAL_HOST_ERROR}
815        ...  '${error_severity}' == 'Predictive' and '${error_creator}' == 'BMC'  ${CMD_PREDICTIVE_ERROR}
816        ...  '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'BMC'  ${CMD_UNRECOVERABLE_ERROR}
817        ...  '${error_severity}' == 'Unrecoverable' and '${error_creator}' == 'HOST'  ${CMD_UNRECOVERABLE_HOST_ERROR}
818        BMC Execute Command  ${cmd}
819    END
820
821
822Get Disk Usage For Error Logs
823    [Documentation]  Get disk usage percentage for error logs.
824
825    ${usage_output}  ${stderr}  ${rc}=  BMC Execute Command  du /var/lib/phosphor-logging/errors
826
827    ${usage_output}=  Fetch From Left  ${usage_output}  \/
828
829    # Convert disk usage unit from KB to MB.
830    ${usage_output}=  Evaluate  ${usage_output} / 1024
831
832    # Logging disk capacity limit is set to 20MB. So calculating the log usage percentage.
833    ${usage_percent}=  Evaluate  ${usage_output} / 20 * 100
834
835    [return]  ${usage_percent}
836
837
838Get PEL Log IDs
839    [Documentation]  Returns the list of PEL log IDs which contains given field's value.
840    [Arguments]  ${pel_section}  ${pel_field}  @{pel_field_value}
841
842    # Description of argument(s):
843    # pel_section      The section of PEL (e.g. Private Header, User Header).
844    # pel_field        The PEL field (e.g. Event Severity, Event Type).
845    # pel_field_value  The list of PEL's field value (e.g. Unrecoverable Error).
846
847    ${pel_ids}=  Get PEL Log Via BMC CLI
848    @{pel_id_list}=  Create List
849
850    FOR  ${id}  IN  @{pel_ids}
851      ${pel_output}=  Peltool  -i ${id}
852      # Example of PEL output from "peltool -i <id>" command.
853      #  [Private Header]:
854      #    [Created at]:                                 08/24/1928 12:04:06
855      #    [Created by]:                                 0x584D
856      #    [Sub-section type]:                           0
857      #    [Entry Id]:                                   0x50000BB7
858      #    [Platform Log Id]:                            0x8200061D
859      #    [CSSVER]:
860      #    [Section Version]:                            1
861      #    [Creator Subsystem]:                          PHYP
862      #    [BMC Event Log Id]:                           341
863      #    [Committed at]:                               03/25/1920 12:06:22
864      #  [User Header]:
865      #    [Log Committed by]:                           0x4552
866      #    [Action Flags]:
867      #      [0]:                                        Report Externally
868      #    [Subsystem]:                                  I/O Subsystem
869      #    [Event Type]:                                 Miscellaneous, Informational Only
870      #    [Sub-section type]:                           0
871      #    [Event Scope]:                                Entire Platform
872      #    [Event Severity]:                             Informational Event
873      #    [Host Transmission]:                          Not Sent
874      #    [Section Version]:                            1
875
876      ${pel_section_output}=  Get From Dictionary  ${pel_output}  ${pel_section}
877      ${pel_field_output}=  Get From Dictionary  ${pel_section_output}  ${pel_field}
878      Run Keyword If  '${pel_field_output}' in @{pel_field_value}  Append To List  ${pel_id_list}  ${id}
879    END
880    Sort List  ${pel_id_list}
881
882    [Return]  ${pel_id_list}
883
884
885Get PEL Log Via BMC CLI
886    [Documentation]  Returns the list of PEL IDs using BMC CLI.
887
888    ${pel_records}=  Peltool  -l
889    ${ids}=  Get Dictionary Keys  ${pel_records}
890    Sort List  ${ids}
891
892    [Return]  ${ids}
893
894
895Get PEL Field Value
896    [Documentation]  Returns the value of given PEL's field.
897    [Arguments]  ${pel_id}  ${pel_section}  ${pel_field}
898
899    # Description of argument(s):
900    # pel_id           The ID of PEL (e.g. 0x5000002D, 0x5000002E).
901    # pel_section      The section of PEL (e.g. Private Header, User Header)
902    # pel_field        The PEL field (e.g. Event Severity, Event Type).
903
904    ${pel_output}=  Peltool  -i ${pel_id}
905
906    # Example of PEL output from "peltool -i <id>" command.
907    #  [Private Header]:
908    #    [Created at]:                                 08/24/1928 12:04:06
909    #    [Created by]:                                 0x584D
910    #    [Sub-section type]:                           0
911    #    [Entry Id]:                                   0x50000BB7
912    #    [Platform Log Id]:                            0x8200061D
913    #    [CSSVER]:
914    #    [Section Version]:                            1
915    #    [Creator Subsystem]:                          PHYP
916    #    [BMC Event Log Id]:                           341
917    #    [Committed at]:                               03/25/1920 12:06:22
918    #  [User Header]:
919    #    [Log Committed by]:                           0x4552
920    #    [Action Flags]:
921    #      [0]:                                        Report Externally
922    #    [Subsystem]:                                  I/O Subsystem
923    #    [Event Type]:                                 Miscellaneous, Informational Only
924    #    [Sub-section type]:                           0
925    #    [Event Scope]:                                Entire Platform
926    #    [Event Severity]:                             Informational Event
927    #    [Host Transmission]:                          Not Sent
928    #    [Section Version]:                            1
929
930    ${pel_section_output}=  Get From Dictionary  ${pel_output}  ${pel_section}
931    ${pel_field_output}=  Get From Dictionary  ${pel_section_output}  ${pel_field}
932
933    [Return]  ${pel_field_output}
934
935
936Verify PEL Transmission To Host
937    [Documentation]  Inject PEL log of given type and verify its host transmission state.
938    [Arguments]  ${error_type}  ${host_state}  ${expected_transmission_state}
939
940    # Description of argument(s):
941    # error_type                         Type of error log to be injected.
942    # host_state                         Host state which is required before error log injection.
943    # expected_transmission_state        Expected host transmission state of PEL log.
944
945    # Get system in required state.
946    Run Keyword If  '${host_state}' == 'Off'
947    ...  Redfish Power Off  stack_mode=skip
948    ...  ELSE IF  '${host_state}' == 'On'
949    ...  Redfish Power On  stack_mode=skip
950
951    Redfish Purge Event Log
952
953    # Inject required error log.
954    Run Keyword If  "${error_type}" == "informational_error"
955    ...  BMC Execute Command  ${CMD_INFORMATIONAL_ERROR}
956    ...  ELSE IF  "${error_type}" == "unrecoverable_error"
957    ...  BMC Execute Command  ${CMD_UNRECOVERABLE_ERROR}
958    ...  ELSE IF  "${error_type}" == "predictive_error"
959    ...  BMC Execute Command  ${CMD_PREDICTIVE_ERROR}
960
961    ${pel_records}=  Get Pel Data From Bmc  True  True
962    ${ids}=  Get Dictionary Keys  ${pel_records}
963    ${pel_id}=  Get From List  ${ids}  -1
964
965    # Check host transmission state for the cases where PEL is
966    # expected to be  offloaded to HOST.
967    Run Keyword If  "${expected_transmission_state}" == "Acked"
968    ...  Wait Until Keyword Succeeds  5 min  10 sec
969    ...  Check If PEL Transmission State Is Expected  ${pel_id}  Acked
970
971    # Adding delay before checking host transmission for the cases where PEL is
972    # not expected to be offloaded to HOST.
973    Run Keyword If  "${expected_transmission_state}" == "Not sent"
974    ...  Run Keywords  Sleep  120s AND
975    ...  Check If PEL Transmission State Is Expected  ${pel_id}  Not sent
976
977
978Check If PEL Transmission State Is Expected
979    [Documentation]  Check if PEL's host transmission state is matching expected state.
980    [Arguments]  ${pel_id}  ${expected_transmission_state}
981
982    # Description of argument(s):
983    # expected_transmission_state       Expected transmission state of PEL log.
984
985    # Check the transmission details in the error log.
986    ${host_transmission}=  Get PEL Field Value  ${pel_id}  User Header  Host Transmission
987    Should Be Equal  ${host_transmission}  ${expected_transmission_state}
988