xref: /openbmc/openbmc-test-automation/lib/pel_utils.py (revision 1e8757aef3fe82f19697e15394f3dfacfb6144c5)
1e7e9171eSGeorge Keishing#!/usr/bin/env python3
2faa5d20aSRahul Maheshwari
3faa5d20aSRahul Maheshwarir"""
4faa5d20aSRahul MaheshwariPEL functions.
5faa5d20aSRahul Maheshwari"""
6faa5d20aSRahul Maheshwari
7faa5d20aSRahul Maheshwariimport json
848ffa2c2SGeorge Keishingimport os
948ffa2c2SGeorge Keishingimport sys
10b813b55aSPatrick Williamsfrom datetime import datetime
11b813b55aSPatrick Williams
1220f38712SPatrick Williamsimport bmc_ssh_utils as bsu
1320f38712SPatrick Williamsimport func_args as fa
1448ffa2c2SGeorge Keishing
1548ffa2c2SGeorge Keishingbase_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
1648ffa2c2SGeorge Keishingsys.path.append(base_path + "/data/")
1748ffa2c2SGeorge Keishing
1809679890SGeorge Keishingimport pel_variables  # NOQA
1937c58c8cSGeorge Keishing
20faa5d20aSRahul Maheshwari
213914da7dSAnusha Dathatriclass PeltoolException(Exception):
226e0e0919SSridevi Ramesh    r"""
236e0e0919SSridevi Ramesh    Base class for peltool related exceptions.
246e0e0919SSridevi Ramesh    """
2548ffa2c2SGeorge Keishing
266e0e0919SSridevi Ramesh    def __init__(self, message):
276e0e0919SSridevi Ramesh        self.message = message
286e0e0919SSridevi Ramesh        super().__init__(self.message)
296e0e0919SSridevi Ramesh
306e0e0919SSridevi Ramesh
31a20876d3SMichael Walshdef peltool(option_string, parse_json=True, **bsu_options):
32faa5d20aSRahul Maheshwari    r"""
33faa5d20aSRahul Maheshwari    Run peltool on the BMC with the caller's option string and return the result.
34faa5d20aSRahul Maheshwari
35faa5d20aSRahul Maheshwari    Example:
36faa5d20aSRahul Maheshwari
37faa5d20aSRahul Maheshwari    ${pel_results}=  Peltool  -l
38faa5d20aSRahul Maheshwari    Rprint Vars  pel_results
39faa5d20aSRahul Maheshwari
40faa5d20aSRahul Maheshwari    pel_results:
41faa5d20aSRahul Maheshwari      [0x50000031]:
42faa5d20aSRahul Maheshwari        [CompID]:                       0x1000
43faa5d20aSRahul Maheshwari        [PLID]:                         0x50000031
44faa5d20aSRahul Maheshwari        [Subsystem]:                    BMC Firmware
45faa5d20aSRahul Maheshwari        [Message]:                      An application had an internal failure
46faa5d20aSRahul Maheshwari        [SRC]:                          BD8D1002
47faa5d20aSRahul Maheshwari        [Commit Time]:                  02/25/2020  04:51:31
48faa5d20aSRahul Maheshwari        [Sev]:                          Unrecoverable Error
49faa5d20aSRahul Maheshwari        [CreatorID]:                    BMC
50faa5d20aSRahul Maheshwari
51faa5d20aSRahul Maheshwari    Description of argument(s):
523914da7dSAnusha Dathatri    option_string                   A string of options which are to be
533914da7dSAnusha Dathatri                                    processed by the peltool command.
543914da7dSAnusha Dathatri    parse_json                      Indicates that the raw JSON data should
553914da7dSAnusha Dathatri                                    parsed into a list of dictionaries.
563914da7dSAnusha Dathatri    bsu_options                     Options to be passed directly to
573914da7dSAnusha Dathatri                                    bmc_execute_command. See its prolog for
58faa5d20aSRahul Maheshwari                                    details.
59faa5d20aSRahul Maheshwari    """
60faa5d20aSRahul Maheshwari
61faa5d20aSRahul Maheshwari    bsu_options = fa.args_to_objects(bsu_options)
623914da7dSAnusha Dathatri    out_buf, _, _ = bsu.bmc_execute_command(
6320f38712SPatrick Williams        "peltool " + option_string, **bsu_options
6420f38712SPatrick Williams    )
65a20876d3SMichael Walsh    if parse_json:
66a20876d3SMichael Walsh        try:
67a20876d3SMichael Walsh            return json.loads(out_buf)
68*1e8757aeSAnusha Dathatri        except ValueError as e:
69*1e8757aeSAnusha Dathatri            if type(out_buf) is str:
70*1e8757aeSAnusha Dathatri                return out_buf
71*1e8757aeSAnusha Dathatri            else:
72*1e8757aeSAnusha Dathatri                print(str(e))
73a20876d3SMichael Walsh                return {}
74faa5d20aSRahul Maheshwari    return out_buf
756e0e0919SSridevi Ramesh
766e0e0919SSridevi Ramesh
7720f38712SPatrick Williamsdef get_pel_data_from_bmc(
7820f38712SPatrick Williams    include_hidden_pels=False, include_informational_pels=False
7920f38712SPatrick Williams):
802930050aSSridevi Ramesh    r"""
812930050aSSridevi Ramesh    Returns PEL data from BMC else throws exception.
82d1cb3252SSridevi Ramesh
83d1cb3252SSridevi Ramesh    Description of arguments:
84d1cb3252SSridevi Ramesh    include_hidden_pels           True/False (default: False).
85d1cb3252SSridevi Ramesh                                  Set True to get hidden PELs else False.
86d1cb3252SSridevi Ramesh    include_informational_pels    True/False (default: False).
87d1cb3252SSridevi Ramesh                                  Set True to get informational PELs else False.
882930050aSSridevi Ramesh    """
892930050aSSridevi Ramesh    try:
90d1cb3252SSridevi Ramesh        pel_cmd = " -l"
91d1cb3252SSridevi Ramesh        if include_hidden_pels:
92d1cb3252SSridevi Ramesh            pel_cmd = pel_cmd + " -h"
93d1cb3252SSridevi Ramesh        if include_informational_pels:
94d1cb3252SSridevi Ramesh            pel_cmd = pel_cmd + " -f"
95d1cb3252SSridevi Ramesh        pel_data = peltool(pel_cmd)
962930050aSSridevi Ramesh        if not pel_data:
972930050aSSridevi Ramesh            print("No PEL data present in BMC ...")
983914da7dSAnusha Dathatri    except Exception as exception:
993914da7dSAnusha Dathatri        raise PeltoolException(
1003914da7dSAnusha Dathatri            "Failed to get PEL data from BMC : " + str(exception)
1013914da7dSAnusha Dathatri        ) from exception
1022930050aSSridevi Ramesh    return pel_data
1032930050aSSridevi Ramesh
1042930050aSSridevi Ramesh
10528cedb13SSushil Singhdef verify_no_pel_exists_on_bmc():
10628cedb13SSushil Singh    r"""
10728cedb13SSushil Singh    Verify that no PEL exists in BMC. Raise an exception if it does.
10828cedb13SSushil Singh    """
10928cedb13SSushil Singh
11028cedb13SSushil Singh    try:
11128cedb13SSushil Singh        pel_data = get_pel_data_from_bmc()
11228cedb13SSushil Singh
11328cedb13SSushil Singh        if len(pel_data) == 0:
11428cedb13SSushil Singh            return True
1153914da7dSAnusha Dathatri
11628cedb13SSushil Singh        print("PEL data present. \n", pel_data)
1173914da7dSAnusha Dathatri        raise PeltoolException("PEL data present in BMC")
1183914da7dSAnusha Dathatri    except Exception as exception:
1193914da7dSAnusha Dathatri        raise PeltoolException(
1203914da7dSAnusha Dathatri            "Failed to get PEL data from BMC : " + str(exception)
1213914da7dSAnusha Dathatri        ) from exception
12228cedb13SSushil Singh
12328cedb13SSushil Singh
12428cedb13SSushil Singhdef compare_pel_and_redfish_event_log(pel_record, event_record):
12528cedb13SSushil Singh    r"""
12628cedb13SSushil Singh    Compare PEL log attributes like "SRC", "Created at" with Redfish
12728cedb13SSushil Singh    event log attributes like "EventId", "Created".
12828cedb13SSushil Singh    Return False if they do not match.
12928cedb13SSushil Singh
13028cedb13SSushil Singh    Description of arguments:
13128cedb13SSushil Singh    pel_record     PEL record.
13228cedb13SSushil Singh    event_record   Redfish event which is equivalent of PEL record.
13328cedb13SSushil Singh    """
13428cedb13SSushil Singh
13528cedb13SSushil Singh    try:
13628cedb13SSushil Singh        # Below is format of PEL record / event record and following
13728cedb13SSushil Singh        # i.e. "SRC", "Created at" from
13828cedb13SSushil Singh        # PEL record is compared with "EventId", "Created" from event record.
13928cedb13SSushil Singh
14028cedb13SSushil Singh        # PEL Log attributes
14128cedb13SSushil Singh        # SRC        : XXXXXXXX
14228cedb13SSushil Singh        # Created at : 11/14/2022 12:38:04
14328cedb13SSushil Singh
14428cedb13SSushil Singh        # Event log attributes
14528cedb13SSushil Singh        # EventId : XXXXXXXX XXXXXXXX XXXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX
14628cedb13SSushil Singh
14728cedb13SSushil Singh        # Created : 2022-11-14T12:38:04+00:00
14828cedb13SSushil Singh
1493914da7dSAnusha Dathatri        print(f"\nPEL records : {pel_record}")
1503914da7dSAnusha Dathatri        print(f"\nEvent records : {event_record}")
15128cedb13SSushil Singh
152b813b55aSPatrick Williams        pel_src = pel_record["pel_data"]["SRC"]
153b813b55aSPatrick Williams        pel_created_time = pel_record["pel_detail_data"]["Private Header"][
154b813b55aSPatrick Williams            "Created at"
155b813b55aSPatrick Williams        ]
15628cedb13SSushil Singh
157b813b55aSPatrick Williams        event_ids = (event_record["EventId"]).split(" ")
15828cedb13SSushil Singh
159b813b55aSPatrick Williams        event_time_format = (event_record["Created"]).split("T")
160b813b55aSPatrick Williams        event_date = (event_time_format[0]).split("-")
161b813b55aSPatrick Williams        event_date = datetime(
162b813b55aSPatrick Williams            int(event_date[0]), int(event_date[1]), int(event_date[2])
163b813b55aSPatrick Williams        )
16428cedb13SSushil Singh        event_date = event_date.strftime("%m/%d/%Y")
165b813b55aSPatrick Williams        event_sub_time_format = (event_time_format[1]).split("+")
16628cedb13SSushil Singh        event_date_time = event_date + " " + event_sub_time_format[0]
16728cedb13SSushil Singh
168b813b55aSPatrick Williams        event_created_time = event_date_time.replace("-", "/")
16928cedb13SSushil Singh
1703914da7dSAnusha Dathatri        print(f"\nPEL SRC : {pel_src} | PEL Created Time : {pel_created_time}")
1713914da7dSAnusha Dathatri
172b813b55aSPatrick Williams        print(
1733914da7dSAnusha Dathatri            f"\nError event ID : {event_ids[0]} | Error Log Created Time "
1743914da7dSAnusha Dathatri            + ": {event_created_time}"
175b813b55aSPatrick Williams        )
17628cedb13SSushil Singh
17728cedb13SSushil Singh        if pel_src == event_ids[0] and pel_created_time == event_created_time:
178b813b55aSPatrick Williams            print(
179b813b55aSPatrick Williams                "\nPEL SRC and created date time match with "
180b813b55aSPatrick Williams                "event ID, created time"
181b813b55aSPatrick Williams            )
18228cedb13SSushil Singh        else:
1833914da7dSAnusha Dathatri            raise PeltoolException(
184b813b55aSPatrick Williams                "\nPEL SRC and created date time did not "
185b813b55aSPatrick Williams                "match with event ID, created time"
186b813b55aSPatrick Williams            )
1873914da7dSAnusha Dathatri    except Exception as exception:
1883914da7dSAnusha Dathatri        raise PeltoolException(
189e16f158fSGeorge Keishing            "Exception occurred during PEL and Event log "
19028cedb13SSushil Singh            "comparison for SRC or event ID and created "
1917899a451SGeorge Keishing            "time : " + str(exception)
1923914da7dSAnusha Dathatri        ) from exception
19328cedb13SSushil Singh
19428cedb13SSushil Singh
195d1cb3252SSridevi Rameshdef fetch_all_pel_ids_for_src(src_id, severity, include_hidden_pels=False):
1966e0e0919SSridevi Ramesh    r"""
197c6dd799dSSridevi Ramesh    Fetch all PEL IDs for the input SRC ID based on the severity type
198c6dd799dSSridevi Ramesh    in the list format.
1996e0e0919SSridevi Ramesh
2006e0e0919SSridevi Ramesh    Description of arguments:
201d1cb3252SSridevi Ramesh    src_id                SRC ID (e.g. BCXXYYYY).
202c6dd799dSSridevi Ramesh    severity              PEL severity (e.g. "Predictive Error"
203c6dd799dSSridevi Ramesh                                             "Recovered Error").
204d1cb3252SSridevi Ramesh    include_hidden_pels   True/False (default: False).
205d1cb3252SSridevi Ramesh                          Set True to get hidden PELs else False.
2066e0e0919SSridevi Ramesh    """
2076e0e0919SSridevi Ramesh
2086e0e0919SSridevi Ramesh    try:
209c6dd799dSSridevi Ramesh        src_pel_ids = []
210d1cb3252SSridevi Ramesh        pel_data = get_pel_data_from_bmc(include_hidden_pels)
2116e0e0919SSridevi Ramesh        pel_id_list = pel_data.keys()
212c6dd799dSSridevi Ramesh        for pel_id in pel_id_list:
213c6dd799dSSridevi Ramesh            # Check if required SRC ID with severity is present
214f3299971SSridevi Ramesh            if src_id in pel_data[pel_id]["SRC"]:
215f3299971SSridevi Ramesh                if pel_data[pel_id]["Sev"] == severity:
216c6dd799dSSridevi Ramesh                    src_pel_ids.append(pel_id)
217c6dd799dSSridevi Ramesh
218c6dd799dSSridevi Ramesh        if not src_pel_ids:
2193914da7dSAnusha Dathatri            raise PeltoolException(
22020f38712SPatrick Williams                src_id + " with severity " + severity + " not present"
22120f38712SPatrick Williams            )
2223914da7dSAnusha Dathatri    except Exception as exception:
2233914da7dSAnusha Dathatri        raise PeltoolException(
2243914da7dSAnusha Dathatri            "Failed to fetch PEL ID for required SRC : " + str(exception)
2253914da7dSAnusha Dathatri        ) from exception
226c6dd799dSSridevi Ramesh    return src_pel_ids
2276e0e0919SSridevi Ramesh
2286e0e0919SSridevi Ramesh
229d1cb3252SSridevi Rameshdef fetch_all_src(include_hidden_pels=False):
2306e0e0919SSridevi Ramesh    r"""
231c6dd799dSSridevi Ramesh    Fetch all SRC IDs from peltool in the list format.
232d1cb3252SSridevi Ramesh
233d1cb3252SSridevi Ramesh    include_hidden_pels       True/False (default: False).
234d1cb3252SSridevi Ramesh                              Set True to get hidden PELs else False.
2356e0e0919SSridevi Ramesh    """
2366e0e0919SSridevi Ramesh    try:
237c6dd799dSSridevi Ramesh        src_id = []
238d1cb3252SSridevi Ramesh        pel_data = get_pel_data_from_bmc(include_hidden_pels)
239c6dd799dSSridevi Ramesh        if pel_data:
2406e0e0919SSridevi Ramesh            pel_id_list = pel_data.keys()
241c6dd799dSSridevi Ramesh            for pel_id in pel_id_list:
242c6dd799dSSridevi Ramesh                src_id.append(pel_data[pel_id]["SRC"])
2436bf23630SSridevi Ramesh        print("SRC IDs: " + str(src_id))
2443914da7dSAnusha Dathatri    except Exception as exception:
2453914da7dSAnusha Dathatri        raise PeltoolException(
2463914da7dSAnusha Dathatri            "Failed to fetch all SRCs : " + str(exception)
2473914da7dSAnusha Dathatri        ) from exception
2486e0e0919SSridevi Ramesh    return src_id
2496bf23630SSridevi Ramesh
2506bf23630SSridevi Ramesh
25120f38712SPatrick Williamsdef check_for_unexpected_src(
2523914da7dSAnusha Dathatri    unexpected_src_list=None, include_hidden_pels=False
25320f38712SPatrick Williams):
2546bf23630SSridevi Ramesh    r"""
2556bf23630SSridevi Ramesh    From the given unexpected SRC list, check if any unexpected SRC created
2566bf23630SSridevi Ramesh    on the BMC. Returns 0 if no SRC found else throws exception.
2576bf23630SSridevi Ramesh
2586bf23630SSridevi Ramesh    Description of arguments:
2596bf23630SSridevi Ramesh    unexpected_src_list       Give unexpected SRCs in the list format.
2606bf23630SSridevi Ramesh                              e.g.: ["BBXXYYYY", "AAXXYYYY"].
261d1cb3252SSridevi Ramesh
262d1cb3252SSridevi Ramesh    include_hidden_pels       True/False (default: False).
263d1cb3252SSridevi Ramesh                              Set True to get hidden PELs else False.
2646bf23630SSridevi Ramesh    """
2656bf23630SSridevi Ramesh    try:
2666bf23630SSridevi Ramesh        unexpected_src_count = 0
2676bf23630SSridevi Ramesh        if not unexpected_src_list:
2686bf23630SSridevi Ramesh            print("Unexpected SRC list is empty.")
269d1cb3252SSridevi Ramesh        src_data = fetch_all_src(include_hidden_pels)
2706bf23630SSridevi Ramesh        for src in unexpected_src_list:
2716bf23630SSridevi Ramesh            if src in src_data:
2726bf23630SSridevi Ramesh                print("Found an unexpected SRC : " + src)
2736bf23630SSridevi Ramesh                unexpected_src_count = unexpected_src_count + 1
27420f38712SPatrick Williams        if unexpected_src_count >= 1:
2753914da7dSAnusha Dathatri            raise PeltoolException("Unexpected SRC found.")
2766bf23630SSridevi Ramesh
2773914da7dSAnusha Dathatri    except Exception as exception:
2783914da7dSAnusha Dathatri        raise PeltoolException(
2793914da7dSAnusha Dathatri            "Failed to verify unexpected SRC list : " + str(exception)
2803914da7dSAnusha Dathatri        ) from exception
2816bf23630SSridevi Ramesh    return unexpected_src_count
282e8801a3fSAnusha Dathatri
283e8801a3fSAnusha Dathatri
284e8801a3fSAnusha Dathatridef filter_unexpected_srcs(expected_srcs=None):
285e8801a3fSAnusha Dathatri    r"""
286e8801a3fSAnusha Dathatri    Return list of SRCs found in BMC after filtering expected SRCs.
287e8801a3fSAnusha Dathatri    If expected_srcs is None then all SRCs found in system are returned.
288e8801a3fSAnusha Dathatri
289e8801a3fSAnusha Dathatri    Description of arguments:
290e8801a3fSAnusha Dathatri    expected_srcs       List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"].
291e8801a3fSAnusha Dathatri    """
292e8801a3fSAnusha Dathatri
293e8801a3fSAnusha Dathatri    srcs_found = fetch_all_src()
294e8801a3fSAnusha Dathatri    if not expected_srcs:
295e8801a3fSAnusha Dathatri        expected_srcs = []
296e8801a3fSAnusha Dathatri    print(expected_srcs)
297e8801a3fSAnusha Dathatri    return list(set(srcs_found) - set(expected_srcs))
2985636ad12SAnusha Dathatri
2995636ad12SAnusha Dathatri
3005636ad12SAnusha Dathatridef get_bmc_event_log_id_for_pel(pel_id):
3015636ad12SAnusha Dathatri    r"""
3025636ad12SAnusha Dathatri    Return BMC event log ID for the given PEL ID.
3035636ad12SAnusha Dathatri
3045636ad12SAnusha Dathatri    Description of arguments:
3055636ad12SAnusha Dathatri    pel_id       PEL ID. E.g. 0x50000021.
3065636ad12SAnusha Dathatri    """
3075636ad12SAnusha Dathatri
3085636ad12SAnusha Dathatri    pel_data = peltool("-i " + pel_id)
3095636ad12SAnusha Dathatri    print(pel_data)
3105636ad12SAnusha Dathatri    bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"]
3115636ad12SAnusha Dathatri    return bmc_id_for_pel
3123914da7dSAnusha Dathatri
3133914da7dSAnusha Dathatri
3143914da7dSAnusha Dathatridef get_latest_pels(number_of_pels=1):
3153914da7dSAnusha Dathatri    r"""
3163914da7dSAnusha Dathatri    Return latest PEL IDs.
3173914da7dSAnusha Dathatri
3183914da7dSAnusha Dathatri    Description of arguments:
3193914da7dSAnusha Dathatri    number_of_pels       Number of PELS to be returned.
3203914da7dSAnusha Dathatri    """
3213914da7dSAnusha Dathatri
3223914da7dSAnusha Dathatri    pel_data = peltool("-lr")
3233914da7dSAnusha Dathatri    pel_ids = list(pel_data.keys())
3243914da7dSAnusha Dathatri    return pel_ids[:number_of_pels]
325