xref: /openbmc/openbmc-test-automation/lib/pel_utils.py (revision 45b45c13c3b888c7a78d6b1ca73dd0289ecba064)
1e7e9171eSGeorge Keishing#!/usr/bin/env python3
2faa5d20aSRahul Maheshwari
3faa5d20aSRahul Maheshwarir"""
4faa5d20aSRahul MaheshwariPEL functions.
5faa5d20aSRahul Maheshwari"""
6faa5d20aSRahul Maheshwari
7faa5d20aSRahul Maheshwariimport json
848ffa2c2SGeorge Keishingimport os
948ffa2c2SGeorge Keishingimport sys
10b813b55aSPatrick Williamsfrom datetime import datetime
11b813b55aSPatrick Williams
1220f38712SPatrick Williamsimport bmc_ssh_utils as bsu
1320f38712SPatrick Williamsimport func_args as fa
1448ffa2c2SGeorge Keishing
1548ffa2c2SGeorge Keishingbase_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
1648ffa2c2SGeorge Keishingsys.path.append(base_path + "/data/")
1748ffa2c2SGeorge Keishing
1809679890SGeorge Keishingimport pel_variables  # NOQA
1937c58c8cSGeorge Keishing
20faa5d20aSRahul Maheshwari
213914da7dSAnusha Dathatriclass PeltoolException(Exception):
226e0e0919SSridevi Ramesh    r"""
236e0e0919SSridevi Ramesh    Base class for peltool related exceptions.
246e0e0919SSridevi Ramesh    """
2548ffa2c2SGeorge Keishing
266e0e0919SSridevi Ramesh    def __init__(self, message):
276e0e0919SSridevi Ramesh        self.message = message
286e0e0919SSridevi Ramesh        super().__init__(self.message)
296e0e0919SSridevi Ramesh
306e0e0919SSridevi Ramesh
31*45b45c13SSridevi Rameshdef peltool(
32*45b45c13SSridevi Ramesh    option_string, peltool_extension=None, parse_json=True, **bsu_options
33*45b45c13SSridevi Ramesh):
34faa5d20aSRahul Maheshwari    r"""
35faa5d20aSRahul Maheshwari    Run peltool on the BMC with the caller's option string and return the result.
36faa5d20aSRahul Maheshwari
37faa5d20aSRahul Maheshwari    Example:
38faa5d20aSRahul Maheshwari
39faa5d20aSRahul Maheshwari    ${pel_results}=  Peltool  -l
40faa5d20aSRahul Maheshwari    Rprint Vars  pel_results
41faa5d20aSRahul Maheshwari
42faa5d20aSRahul Maheshwari    pel_results:
43faa5d20aSRahul Maheshwari      [0x50000031]:
44faa5d20aSRahul Maheshwari        [CompID]:                       0x1000
45faa5d20aSRahul Maheshwari        [PLID]:                         0x50000031
46faa5d20aSRahul Maheshwari        [Subsystem]:                    BMC Firmware
47faa5d20aSRahul Maheshwari        [Message]:                      An application had an internal failure
48faa5d20aSRahul Maheshwari        [SRC]:                          BD8D1002
49faa5d20aSRahul Maheshwari        [Commit Time]:                  02/25/2020  04:51:31
50faa5d20aSRahul Maheshwari        [Sev]:                          Unrecoverable Error
51faa5d20aSRahul Maheshwari        [CreatorID]:                    BMC
52faa5d20aSRahul Maheshwari
53faa5d20aSRahul Maheshwari    Description of argument(s):
543914da7dSAnusha Dathatri    option_string                   A string of options which are to be
553914da7dSAnusha Dathatri                                    processed by the peltool command.
56*45b45c13SSridevi Ramesh    peltool_extension               Provide peltool extension format.
57*45b45c13SSridevi Ramesh                                    Default: None.
583914da7dSAnusha Dathatri    parse_json                      Indicates that the raw JSON data should
593914da7dSAnusha Dathatri                                    parsed into a list of dictionaries.
603914da7dSAnusha Dathatri    bsu_options                     Options to be passed directly to
613914da7dSAnusha Dathatri                                    bmc_execute_command. See its prolog for
62faa5d20aSRahul Maheshwari                                    details.
63faa5d20aSRahul Maheshwari    """
64faa5d20aSRahul Maheshwari
65faa5d20aSRahul Maheshwari    bsu_options = fa.args_to_objects(bsu_options)
66*45b45c13SSridevi Ramesh    peltool_cmd = "peltool"
67*45b45c13SSridevi Ramesh    if peltool_extension:
68*45b45c13SSridevi Ramesh        peltool_cmd = peltool_cmd + peltool_extension
69*45b45c13SSridevi Ramesh
703914da7dSAnusha Dathatri    out_buf, _, _ = bsu.bmc_execute_command(
71*45b45c13SSridevi Ramesh        peltool_cmd + " " + option_string, **bsu_options
7220f38712SPatrick Williams    )
73a20876d3SMichael Walsh    if parse_json:
74a20876d3SMichael Walsh        try:
75a20876d3SMichael Walsh            return json.loads(out_buf)
761e8757aeSAnusha Dathatri        except ValueError as e:
771e8757aeSAnusha Dathatri            if type(out_buf) is str:
781e8757aeSAnusha Dathatri                return out_buf
791e8757aeSAnusha Dathatri            else:
801e8757aeSAnusha Dathatri                print(str(e))
81a20876d3SMichael Walsh                return {}
82faa5d20aSRahul Maheshwari    return out_buf
836e0e0919SSridevi Ramesh
846e0e0919SSridevi Ramesh
8520f38712SPatrick Williamsdef get_pel_data_from_bmc(
8620f38712SPatrick Williams    include_hidden_pels=False, include_informational_pels=False
8720f38712SPatrick Williams):
882930050aSSridevi Ramesh    r"""
892930050aSSridevi Ramesh    Returns PEL data from BMC else throws exception.
90d1cb3252SSridevi Ramesh
91d1cb3252SSridevi Ramesh    Description of arguments:
92d1cb3252SSridevi Ramesh    include_hidden_pels           True/False (default: False).
93d1cb3252SSridevi Ramesh                                  Set True to get hidden PELs else False.
94d1cb3252SSridevi Ramesh    include_informational_pels    True/False (default: False).
95d1cb3252SSridevi Ramesh                                  Set True to get informational PELs else False.
962930050aSSridevi Ramesh    """
972930050aSSridevi Ramesh    try:
98d1cb3252SSridevi Ramesh        pel_cmd = " -l"
99d1cb3252SSridevi Ramesh        if include_hidden_pels:
100d1cb3252SSridevi Ramesh            pel_cmd = pel_cmd + " -h"
101d1cb3252SSridevi Ramesh        if include_informational_pels:
102d1cb3252SSridevi Ramesh            pel_cmd = pel_cmd + " -f"
103d1cb3252SSridevi Ramesh        pel_data = peltool(pel_cmd)
1042930050aSSridevi Ramesh        if not pel_data:
1052930050aSSridevi Ramesh            print("No PEL data present in BMC ...")
1063914da7dSAnusha Dathatri    except Exception as exception:
1073914da7dSAnusha Dathatri        raise PeltoolException(
1083914da7dSAnusha Dathatri            "Failed to get PEL data from BMC : " + str(exception)
1093914da7dSAnusha Dathatri        ) from exception
1102930050aSSridevi Ramesh    return pel_data
1112930050aSSridevi Ramesh
1122930050aSSridevi Ramesh
11328cedb13SSushil Singhdef verify_no_pel_exists_on_bmc():
11428cedb13SSushil Singh    r"""
11528cedb13SSushil Singh    Verify that no PEL exists in BMC. Raise an exception if it does.
11628cedb13SSushil Singh    """
11728cedb13SSushil Singh
11828cedb13SSushil Singh    try:
11928cedb13SSushil Singh        pel_data = get_pel_data_from_bmc()
12028cedb13SSushil Singh
12128cedb13SSushil Singh        if len(pel_data) == 0:
12228cedb13SSushil Singh            return True
1233914da7dSAnusha Dathatri
12428cedb13SSushil Singh        print("PEL data present. \n", pel_data)
1253914da7dSAnusha Dathatri        raise PeltoolException("PEL data present in BMC")
1263914da7dSAnusha Dathatri    except Exception as exception:
1273914da7dSAnusha Dathatri        raise PeltoolException(
1283914da7dSAnusha Dathatri            "Failed to get PEL data from BMC : " + str(exception)
1293914da7dSAnusha Dathatri        ) from exception
13028cedb13SSushil Singh
13128cedb13SSushil Singh
13228cedb13SSushil Singhdef compare_pel_and_redfish_event_log(pel_record, event_record):
13328cedb13SSushil Singh    r"""
13428cedb13SSushil Singh    Compare PEL log attributes like "SRC", "Created at" with Redfish
13528cedb13SSushil Singh    event log attributes like "EventId", "Created".
13628cedb13SSushil Singh    Return False if they do not match.
13728cedb13SSushil Singh
13828cedb13SSushil Singh    Description of arguments:
13928cedb13SSushil Singh    pel_record     PEL record.
14028cedb13SSushil Singh    event_record   Redfish event which is equivalent of PEL record.
14128cedb13SSushil Singh    """
14228cedb13SSushil Singh
14328cedb13SSushil Singh    try:
14428cedb13SSushil Singh        # Below is format of PEL record / event record and following
14528cedb13SSushil Singh        # i.e. "SRC", "Created at" from
14628cedb13SSushil Singh        # PEL record is compared with "EventId", "Created" from event record.
14728cedb13SSushil Singh
14828cedb13SSushil Singh        # PEL Log attributes
14928cedb13SSushil Singh        # SRC        : XXXXXXXX
15028cedb13SSushil Singh        # Created at : 11/14/2022 12:38:04
15128cedb13SSushil Singh
15228cedb13SSushil Singh        # Event log attributes
15328cedb13SSushil Singh        # EventId : XXXXXXXX XXXXXXXX XXXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX
15428cedb13SSushil Singh
15528cedb13SSushil Singh        # Created : 2022-11-14T12:38:04+00:00
15628cedb13SSushil Singh
1573914da7dSAnusha Dathatri        print(f"\nPEL records : {pel_record}")
1583914da7dSAnusha Dathatri        print(f"\nEvent records : {event_record}")
15928cedb13SSushil Singh
160b813b55aSPatrick Williams        pel_src = pel_record["pel_data"]["SRC"]
161b813b55aSPatrick Williams        pel_created_time = pel_record["pel_detail_data"]["Private Header"][
162b813b55aSPatrick Williams            "Created at"
163b813b55aSPatrick Williams        ]
16428cedb13SSushil Singh
165b813b55aSPatrick Williams        event_ids = (event_record["EventId"]).split(" ")
16628cedb13SSushil Singh
167b813b55aSPatrick Williams        event_time_format = (event_record["Created"]).split("T")
168b813b55aSPatrick Williams        event_date = (event_time_format[0]).split("-")
169b813b55aSPatrick Williams        event_date = datetime(
170b813b55aSPatrick Williams            int(event_date[0]), int(event_date[1]), int(event_date[2])
171b813b55aSPatrick Williams        )
17228cedb13SSushil Singh        event_date = event_date.strftime("%m/%d/%Y")
173b813b55aSPatrick Williams        event_sub_time_format = (event_time_format[1]).split("+")
17428cedb13SSushil Singh        event_date_time = event_date + " " + event_sub_time_format[0]
17528cedb13SSushil Singh
176b813b55aSPatrick Williams        event_created_time = event_date_time.replace("-", "/")
17728cedb13SSushil Singh
1783914da7dSAnusha Dathatri        print(f"\nPEL SRC : {pel_src} | PEL Created Time : {pel_created_time}")
1793914da7dSAnusha Dathatri
180b813b55aSPatrick Williams        print(
1813914da7dSAnusha Dathatri            f"\nError event ID : {event_ids[0]} | Error Log Created Time "
1823914da7dSAnusha Dathatri            + ": {event_created_time}"
183b813b55aSPatrick Williams        )
18428cedb13SSushil Singh
18528cedb13SSushil Singh        if pel_src == event_ids[0] and pel_created_time == event_created_time:
186b813b55aSPatrick Williams            print(
187b813b55aSPatrick Williams                "\nPEL SRC and created date time match with "
188b813b55aSPatrick Williams                "event ID, created time"
189b813b55aSPatrick Williams            )
19028cedb13SSushil Singh        else:
1913914da7dSAnusha Dathatri            raise PeltoolException(
192b813b55aSPatrick Williams                "\nPEL SRC and created date time did not "
193b813b55aSPatrick Williams                "match with event ID, created time"
194b813b55aSPatrick Williams            )
1953914da7dSAnusha Dathatri    except Exception as exception:
1963914da7dSAnusha Dathatri        raise PeltoolException(
197e16f158fSGeorge Keishing            "Exception occurred during PEL and Event log "
19828cedb13SSushil Singh            "comparison for SRC or event ID and created "
1997899a451SGeorge Keishing            "time : " + str(exception)
2003914da7dSAnusha Dathatri        ) from exception
20128cedb13SSushil Singh
20228cedb13SSushil Singh
203d1cb3252SSridevi Rameshdef fetch_all_pel_ids_for_src(src_id, severity, include_hidden_pels=False):
2046e0e0919SSridevi Ramesh    r"""
205c6dd799dSSridevi Ramesh    Fetch all PEL IDs for the input SRC ID based on the severity type
206c6dd799dSSridevi Ramesh    in the list format.
2076e0e0919SSridevi Ramesh
2086e0e0919SSridevi Ramesh    Description of arguments:
209d1cb3252SSridevi Ramesh    src_id                SRC ID (e.g. BCXXYYYY).
210c6dd799dSSridevi Ramesh    severity              PEL severity (e.g. "Predictive Error"
211c6dd799dSSridevi Ramesh                                             "Recovered Error").
212d1cb3252SSridevi Ramesh    include_hidden_pels   True/False (default: False).
213d1cb3252SSridevi Ramesh                          Set True to get hidden PELs else False.
2146e0e0919SSridevi Ramesh    """
2156e0e0919SSridevi Ramesh
2166e0e0919SSridevi Ramesh    try:
217c6dd799dSSridevi Ramesh        src_pel_ids = []
218d1cb3252SSridevi Ramesh        pel_data = get_pel_data_from_bmc(include_hidden_pels)
2196e0e0919SSridevi Ramesh        pel_id_list = pel_data.keys()
220c6dd799dSSridevi Ramesh        for pel_id in pel_id_list:
221c6dd799dSSridevi Ramesh            # Check if required SRC ID with severity is present
222f3299971SSridevi Ramesh            if src_id in pel_data[pel_id]["SRC"]:
223f3299971SSridevi Ramesh                if pel_data[pel_id]["Sev"] == severity:
224c6dd799dSSridevi Ramesh                    src_pel_ids.append(pel_id)
225c6dd799dSSridevi Ramesh
226c6dd799dSSridevi Ramesh        if not src_pel_ids:
2273914da7dSAnusha Dathatri            raise PeltoolException(
22820f38712SPatrick Williams                src_id + " with severity " + severity + " not present"
22920f38712SPatrick Williams            )
2303914da7dSAnusha Dathatri    except Exception as exception:
2313914da7dSAnusha Dathatri        raise PeltoolException(
2323914da7dSAnusha Dathatri            "Failed to fetch PEL ID for required SRC : " + str(exception)
2333914da7dSAnusha Dathatri        ) from exception
234c6dd799dSSridevi Ramesh    return src_pel_ids
2356e0e0919SSridevi Ramesh
2366e0e0919SSridevi Ramesh
237d1cb3252SSridevi Rameshdef fetch_all_src(include_hidden_pels=False):
2386e0e0919SSridevi Ramesh    r"""
239c6dd799dSSridevi Ramesh    Fetch all SRC IDs from peltool in the list format.
240d1cb3252SSridevi Ramesh
241d1cb3252SSridevi Ramesh    include_hidden_pels       True/False (default: False).
242d1cb3252SSridevi Ramesh                              Set True to get hidden PELs else False.
2436e0e0919SSridevi Ramesh    """
2446e0e0919SSridevi Ramesh    try:
245c6dd799dSSridevi Ramesh        src_id = []
246d1cb3252SSridevi Ramesh        pel_data = get_pel_data_from_bmc(include_hidden_pels)
247c6dd799dSSridevi Ramesh        if pel_data:
2486e0e0919SSridevi Ramesh            pel_id_list = pel_data.keys()
249c6dd799dSSridevi Ramesh            for pel_id in pel_id_list:
250c6dd799dSSridevi Ramesh                src_id.append(pel_data[pel_id]["SRC"])
2516bf23630SSridevi Ramesh        print("SRC IDs: " + str(src_id))
2523914da7dSAnusha Dathatri    except Exception as exception:
2533914da7dSAnusha Dathatri        raise PeltoolException(
2543914da7dSAnusha Dathatri            "Failed to fetch all SRCs : " + str(exception)
2553914da7dSAnusha Dathatri        ) from exception
2566e0e0919SSridevi Ramesh    return src_id
2576bf23630SSridevi Ramesh
2586bf23630SSridevi Ramesh
25920f38712SPatrick Williamsdef check_for_unexpected_src(
2603914da7dSAnusha Dathatri    unexpected_src_list=None, include_hidden_pels=False
26120f38712SPatrick Williams):
2626bf23630SSridevi Ramesh    r"""
2636bf23630SSridevi Ramesh    From the given unexpected SRC list, check if any unexpected SRC created
2646bf23630SSridevi Ramesh    on the BMC. Returns 0 if no SRC found else throws exception.
2656bf23630SSridevi Ramesh
2666bf23630SSridevi Ramesh    Description of arguments:
2676bf23630SSridevi Ramesh    unexpected_src_list       Give unexpected SRCs in the list format.
2686bf23630SSridevi Ramesh                              e.g.: ["BBXXYYYY", "AAXXYYYY"].
269d1cb3252SSridevi Ramesh
270d1cb3252SSridevi Ramesh    include_hidden_pels       True/False (default: False).
271d1cb3252SSridevi Ramesh                              Set True to get hidden PELs else False.
2726bf23630SSridevi Ramesh    """
2736bf23630SSridevi Ramesh    try:
2746bf23630SSridevi Ramesh        unexpected_src_count = 0
2756bf23630SSridevi Ramesh        if not unexpected_src_list:
2766bf23630SSridevi Ramesh            print("Unexpected SRC list is empty.")
277d1cb3252SSridevi Ramesh        src_data = fetch_all_src(include_hidden_pels)
2786bf23630SSridevi Ramesh        for src in unexpected_src_list:
2796bf23630SSridevi Ramesh            if src in src_data:
2806bf23630SSridevi Ramesh                print("Found an unexpected SRC : " + src)
2816bf23630SSridevi Ramesh                unexpected_src_count = unexpected_src_count + 1
28220f38712SPatrick Williams        if unexpected_src_count >= 1:
2833914da7dSAnusha Dathatri            raise PeltoolException("Unexpected SRC found.")
2846bf23630SSridevi Ramesh
2853914da7dSAnusha Dathatri    except Exception as exception:
2863914da7dSAnusha Dathatri        raise PeltoolException(
2873914da7dSAnusha Dathatri            "Failed to verify unexpected SRC list : " + str(exception)
2883914da7dSAnusha Dathatri        ) from exception
2896bf23630SSridevi Ramesh    return unexpected_src_count
290e8801a3fSAnusha Dathatri
291e8801a3fSAnusha Dathatri
292e8801a3fSAnusha Dathatridef filter_unexpected_srcs(expected_srcs=None):
293e8801a3fSAnusha Dathatri    r"""
294e8801a3fSAnusha Dathatri    Return list of SRCs found in BMC after filtering expected SRCs.
295e8801a3fSAnusha Dathatri    If expected_srcs is None then all SRCs found in system are returned.
296e8801a3fSAnusha Dathatri
297e8801a3fSAnusha Dathatri    Description of arguments:
298e8801a3fSAnusha Dathatri    expected_srcs       List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"].
299e8801a3fSAnusha Dathatri    """
300e8801a3fSAnusha Dathatri
301e8801a3fSAnusha Dathatri    srcs_found = fetch_all_src()
302e8801a3fSAnusha Dathatri    if not expected_srcs:
303e8801a3fSAnusha Dathatri        expected_srcs = []
304e8801a3fSAnusha Dathatri    print(expected_srcs)
305e8801a3fSAnusha Dathatri    return list(set(srcs_found) - set(expected_srcs))
3065636ad12SAnusha Dathatri
3075636ad12SAnusha Dathatri
3085636ad12SAnusha Dathatridef get_bmc_event_log_id_for_pel(pel_id):
3095636ad12SAnusha Dathatri    r"""
3105636ad12SAnusha Dathatri    Return BMC event log ID for the given PEL ID.
3115636ad12SAnusha Dathatri
3125636ad12SAnusha Dathatri    Description of arguments:
3135636ad12SAnusha Dathatri    pel_id       PEL ID. E.g. 0x50000021.
3145636ad12SAnusha Dathatri    """
3155636ad12SAnusha Dathatri
3165636ad12SAnusha Dathatri    pel_data = peltool("-i " + pel_id)
3175636ad12SAnusha Dathatri    print(pel_data)
3185636ad12SAnusha Dathatri    bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"]
3195636ad12SAnusha Dathatri    return bmc_id_for_pel
3203914da7dSAnusha Dathatri
3213914da7dSAnusha Dathatri
3223914da7dSAnusha Dathatridef get_latest_pels(number_of_pels=1):
3233914da7dSAnusha Dathatri    r"""
3243914da7dSAnusha Dathatri    Return latest PEL IDs.
3253914da7dSAnusha Dathatri
3263914da7dSAnusha Dathatri    Description of arguments:
3273914da7dSAnusha Dathatri    number_of_pels       Number of PELS to be returned.
3283914da7dSAnusha Dathatri    """
3293914da7dSAnusha Dathatri
3303914da7dSAnusha Dathatri    pel_data = peltool("-lr")
3313914da7dSAnusha Dathatri    pel_ids = list(pel_data.keys())
3323914da7dSAnusha Dathatri    return pel_ids[:number_of_pels]
333