1e7e9171eSGeorge Keishing#!/usr/bin/env python3 2faa5d20aSRahul Maheshwari 3faa5d20aSRahul Maheshwarir""" 4faa5d20aSRahul MaheshwariPEL functions. 5faa5d20aSRahul Maheshwari""" 6faa5d20aSRahul Maheshwari 7faa5d20aSRahul Maheshwariimport json 848ffa2c2SGeorge Keishingimport os 948ffa2c2SGeorge Keishingimport sys 10b813b55aSPatrick Williamsfrom datetime import datetime 11b813b55aSPatrick Williams 1220f38712SPatrick Williamsimport bmc_ssh_utils as bsu 1320f38712SPatrick Williamsimport func_args as fa 1448ffa2c2SGeorge Keishing 1548ffa2c2SGeorge Keishingbase_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 1648ffa2c2SGeorge Keishingsys.path.append(base_path + "/data/") 1748ffa2c2SGeorge Keishing 1809679890SGeorge Keishingimport pel_variables # NOQA 1937c58c8cSGeorge Keishing 20faa5d20aSRahul Maheshwari 216e0e0919SSridevi Rameshclass peltool_exception(Exception): 226e0e0919SSridevi Ramesh r""" 236e0e0919SSridevi Ramesh Base class for peltool related exceptions. 246e0e0919SSridevi Ramesh """ 2548ffa2c2SGeorge Keishing 266e0e0919SSridevi Ramesh def __init__(self, message): 276e0e0919SSridevi Ramesh self.message = message 286e0e0919SSridevi Ramesh super().__init__(self.message) 296e0e0919SSridevi Ramesh 306e0e0919SSridevi Ramesh 31a20876d3SMichael Walshdef peltool(option_string, parse_json=True, **bsu_options): 32faa5d20aSRahul Maheshwari r""" 33faa5d20aSRahul Maheshwari Run peltool on the BMC with the caller's option string and return the result. 34faa5d20aSRahul Maheshwari 35faa5d20aSRahul Maheshwari Example: 36faa5d20aSRahul Maheshwari 37faa5d20aSRahul Maheshwari ${pel_results}= Peltool -l 38faa5d20aSRahul Maheshwari Rprint Vars pel_results 39faa5d20aSRahul Maheshwari 40faa5d20aSRahul Maheshwari pel_results: 41faa5d20aSRahul Maheshwari [0x50000031]: 42faa5d20aSRahul Maheshwari [CompID]: 0x1000 43faa5d20aSRahul Maheshwari [PLID]: 0x50000031 44faa5d20aSRahul Maheshwari [Subsystem]: BMC Firmware 45faa5d20aSRahul Maheshwari [Message]: An application had an internal failure 46faa5d20aSRahul Maheshwari [SRC]: BD8D1002 47faa5d20aSRahul Maheshwari [Commit Time]: 02/25/2020 04:51:31 48faa5d20aSRahul Maheshwari [Sev]: Unrecoverable Error 49faa5d20aSRahul Maheshwari [CreatorID]: BMC 50faa5d20aSRahul Maheshwari 51faa5d20aSRahul Maheshwari Description of argument(s): 52faa5d20aSRahul Maheshwari option_string A string of options which are to be processed by the peltool command. 53a20876d3SMichael Walsh parse_json Indicates that the raw JSON data should parsed into a list of 54a20876d3SMichael Walsh dictionaries. 55faa5d20aSRahul Maheshwari bsu_options Options to be passed directly to bmc_execute_command. See its prolog for 56faa5d20aSRahul Maheshwari details. 57faa5d20aSRahul Maheshwari """ 58faa5d20aSRahul Maheshwari 59faa5d20aSRahul Maheshwari bsu_options = fa.args_to_objects(bsu_options) 6020f38712SPatrick Williams out_buf, stderr, rc = bsu.bmc_execute_command( 6120f38712SPatrick Williams "peltool " + option_string, **bsu_options 6220f38712SPatrick Williams ) 63a20876d3SMichael Walsh if parse_json: 64a20876d3SMichael Walsh try: 65a20876d3SMichael Walsh return json.loads(out_buf) 66c6dd799dSSridevi Ramesh except ValueError: 67a20876d3SMichael Walsh return {} 68faa5d20aSRahul Maheshwari return out_buf 696e0e0919SSridevi Ramesh 706e0e0919SSridevi Ramesh 7120f38712SPatrick Williamsdef get_pel_data_from_bmc( 7220f38712SPatrick Williams include_hidden_pels=False, include_informational_pels=False 7320f38712SPatrick Williams): 742930050aSSridevi Ramesh r""" 752930050aSSridevi Ramesh Returns PEL data from BMC else throws exception. 76d1cb3252SSridevi Ramesh 77d1cb3252SSridevi Ramesh Description of arguments: 78d1cb3252SSridevi Ramesh include_hidden_pels True/False (default: False). 79d1cb3252SSridevi Ramesh Set True to get hidden PELs else False. 80d1cb3252SSridevi Ramesh include_informational_pels True/False (default: False). 81d1cb3252SSridevi Ramesh Set True to get informational PELs else False. 822930050aSSridevi Ramesh """ 832930050aSSridevi Ramesh try: 84d1cb3252SSridevi Ramesh pel_cmd = " -l" 85d1cb3252SSridevi Ramesh if include_hidden_pels: 86d1cb3252SSridevi Ramesh pel_cmd = pel_cmd + " -h" 87d1cb3252SSridevi Ramesh if include_informational_pels: 88d1cb3252SSridevi Ramesh pel_cmd = pel_cmd + " -f" 89d1cb3252SSridevi Ramesh pel_data = peltool(pel_cmd) 902930050aSSridevi Ramesh if not pel_data: 912930050aSSridevi Ramesh print("No PEL data present in BMC ...") 922930050aSSridevi Ramesh except Exception as e: 932930050aSSridevi Ramesh raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) 942930050aSSridevi Ramesh return pel_data 952930050aSSridevi Ramesh 962930050aSSridevi Ramesh 9728cedb13SSushil Singhdef verify_no_pel_exists_on_bmc(): 9828cedb13SSushil Singh r""" 9928cedb13SSushil Singh Verify that no PEL exists in BMC. Raise an exception if it does. 10028cedb13SSushil Singh """ 10128cedb13SSushil Singh 10228cedb13SSushil Singh try: 10328cedb13SSushil Singh pel_data = get_pel_data_from_bmc() 10428cedb13SSushil Singh 10528cedb13SSushil Singh if len(pel_data) == 0: 10628cedb13SSushil Singh return True 10728cedb13SSushil Singh else: 10828cedb13SSushil Singh print("PEL data present. \n", pel_data) 10928cedb13SSushil Singh raise peltool_exception("PEL data present in BMC") 11028cedb13SSushil Singh except Exception as e: 11128cedb13SSushil Singh raise peltool_exception("Failed to get PEL data from BMC : " + str(e)) 11228cedb13SSushil Singh 11328cedb13SSushil Singh 11428cedb13SSushil Singhdef compare_pel_and_redfish_event_log(pel_record, event_record): 11528cedb13SSushil Singh r""" 11628cedb13SSushil Singh Compare PEL log attributes like "SRC", "Created at" with Redfish 11728cedb13SSushil Singh event log attributes like "EventId", "Created". 11828cedb13SSushil Singh Return False if they do not match. 11928cedb13SSushil Singh 12028cedb13SSushil Singh Description of arguments: 12128cedb13SSushil Singh pel_record PEL record. 12228cedb13SSushil Singh event_record Redfish event which is equivalent of PEL record. 12328cedb13SSushil Singh """ 12428cedb13SSushil Singh 12528cedb13SSushil Singh try: 12628cedb13SSushil Singh # Below is format of PEL record / event record and following 12728cedb13SSushil Singh # i.e. "SRC", "Created at" from 12828cedb13SSushil Singh # PEL record is compared with "EventId", "Created" from event record. 12928cedb13SSushil Singh 13028cedb13SSushil Singh # PEL Log attributes 13128cedb13SSushil Singh # SRC : XXXXXXXX 13228cedb13SSushil Singh # Created at : 11/14/2022 12:38:04 13328cedb13SSushil Singh 13428cedb13SSushil Singh # Event log attributes 13528cedb13SSushil Singh # EventId : XXXXXXXX XXXXXXXX XXXXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX 13628cedb13SSushil Singh 13728cedb13SSushil Singh # Created : 2022-11-14T12:38:04+00:00 13828cedb13SSushil Singh 13928cedb13SSushil Singh print("\nPEL records : {0}".format(pel_record)) 14028cedb13SSushil Singh print("\nEvent records : {0}".format(event_record)) 14128cedb13SSushil Singh 142b813b55aSPatrick Williams pel_src = pel_record["pel_data"]["SRC"] 143b813b55aSPatrick Williams pel_created_time = pel_record["pel_detail_data"]["Private Header"][ 144b813b55aSPatrick Williams "Created at" 145b813b55aSPatrick Williams ] 14628cedb13SSushil Singh 147b813b55aSPatrick Williams event_ids = (event_record["EventId"]).split(" ") 14828cedb13SSushil Singh 149b813b55aSPatrick Williams event_time_format = (event_record["Created"]).split("T") 150b813b55aSPatrick Williams event_date = (event_time_format[0]).split("-") 151b813b55aSPatrick Williams event_date = datetime( 152b813b55aSPatrick Williams int(event_date[0]), int(event_date[1]), int(event_date[2]) 153b813b55aSPatrick Williams ) 15428cedb13SSushil Singh event_date = event_date.strftime("%m/%d/%Y") 155b813b55aSPatrick Williams event_sub_time_format = (event_time_format[1]).split("+") 15628cedb13SSushil Singh event_date_time = event_date + " " + event_sub_time_format[0] 15728cedb13SSushil Singh 158b813b55aSPatrick Williams event_created_time = event_date_time.replace("-", "/") 15928cedb13SSushil Singh 160b813b55aSPatrick Williams print( 161b813b55aSPatrick Williams "\nPEL SRC : {0} | PEL Created Time : {1}".format( 162b813b55aSPatrick Williams pel_src, pel_created_time 163b813b55aSPatrick Williams ) 164b813b55aSPatrick Williams ) 165b813b55aSPatrick Williams print( 166b813b55aSPatrick Williams "\nError event ID : {0} | Error Log Created Time : {1}".format( 167b813b55aSPatrick Williams event_ids[0], event_created_time 168b813b55aSPatrick Williams ) 169b813b55aSPatrick Williams ) 17028cedb13SSushil Singh 17128cedb13SSushil Singh if pel_src == event_ids[0] and pel_created_time == event_created_time: 172b813b55aSPatrick Williams print( 173b813b55aSPatrick Williams "\nPEL SRC and created date time match with " 174b813b55aSPatrick Williams "event ID, created time" 175b813b55aSPatrick Williams ) 17628cedb13SSushil Singh else: 177b813b55aSPatrick Williams raise peltool_exception( 178b813b55aSPatrick Williams "\nPEL SRC and created date time did not " 179b813b55aSPatrick Williams "match with event ID, created time" 180b813b55aSPatrick Williams ) 18128cedb13SSushil Singh except Exception as e: 182b813b55aSPatrick Williams raise peltool_exception( 183e16f158fSGeorge Keishing "Exception occurred during PEL and Event log " 18428cedb13SSushil Singh "comparison for SRC or event ID and created " 185b813b55aSPatrick Williams "time : " 186b813b55aSPatrick Williams + str(e) 187b813b55aSPatrick Williams ) 18828cedb13SSushil Singh 18928cedb13SSushil Singh 190d1cb3252SSridevi Rameshdef fetch_all_pel_ids_for_src(src_id, severity, include_hidden_pels=False): 1916e0e0919SSridevi Ramesh r""" 192c6dd799dSSridevi Ramesh Fetch all PEL IDs for the input SRC ID based on the severity type 193c6dd799dSSridevi Ramesh in the list format. 1946e0e0919SSridevi Ramesh 1956e0e0919SSridevi Ramesh Description of arguments: 196d1cb3252SSridevi Ramesh src_id SRC ID (e.g. BCXXYYYY). 197c6dd799dSSridevi Ramesh severity PEL severity (e.g. "Predictive Error" 198c6dd799dSSridevi Ramesh "Recovered Error"). 199d1cb3252SSridevi Ramesh include_hidden_pels True/False (default: False). 200d1cb3252SSridevi Ramesh Set True to get hidden PELs else False. 2016e0e0919SSridevi Ramesh """ 2026e0e0919SSridevi Ramesh 2036e0e0919SSridevi Ramesh try: 204c6dd799dSSridevi Ramesh src_pel_ids = [] 205d1cb3252SSridevi Ramesh pel_data = get_pel_data_from_bmc(include_hidden_pels) 2066e0e0919SSridevi Ramesh pel_id_list = pel_data.keys() 207c6dd799dSSridevi Ramesh for pel_id in pel_id_list: 208c6dd799dSSridevi Ramesh # Check if required SRC ID with severity is present 209*f3299971SSridevi Ramesh if src_id in pel_data[pel_id]["SRC"]: 210*f3299971SSridevi Ramesh if pel_data[pel_id]["Sev"] == severity: 211c6dd799dSSridevi Ramesh src_pel_ids.append(pel_id) 212c6dd799dSSridevi Ramesh 213c6dd799dSSridevi Ramesh if not src_pel_ids: 21420f38712SPatrick Williams raise peltool_exception( 21520f38712SPatrick Williams src_id + " with severity " + severity + " not present" 21620f38712SPatrick Williams ) 2176e0e0919SSridevi Ramesh except Exception as e: 21820f38712SPatrick Williams raise peltool_exception( 21920f38712SPatrick Williams "Failed to fetch PEL ID for required SRC : " + str(e) 22020f38712SPatrick Williams ) 221c6dd799dSSridevi Ramesh return src_pel_ids 2226e0e0919SSridevi Ramesh 2236e0e0919SSridevi Ramesh 224d1cb3252SSridevi Rameshdef fetch_all_src(include_hidden_pels=False): 2256e0e0919SSridevi Ramesh r""" 226c6dd799dSSridevi Ramesh Fetch all SRC IDs from peltool in the list format. 227d1cb3252SSridevi Ramesh 228d1cb3252SSridevi Ramesh include_hidden_pels True/False (default: False). 229d1cb3252SSridevi Ramesh Set True to get hidden PELs else False. 2306e0e0919SSridevi Ramesh """ 2316e0e0919SSridevi Ramesh try: 232c6dd799dSSridevi Ramesh src_id = [] 233d1cb3252SSridevi Ramesh pel_data = get_pel_data_from_bmc(include_hidden_pels) 234c6dd799dSSridevi Ramesh if pel_data: 2356e0e0919SSridevi Ramesh pel_id_list = pel_data.keys() 236c6dd799dSSridevi Ramesh for pel_id in pel_id_list: 237c6dd799dSSridevi Ramesh src_id.append(pel_data[pel_id]["SRC"]) 2386bf23630SSridevi Ramesh print("SRC IDs: " + str(src_id)) 2396e0e0919SSridevi Ramesh except Exception as e: 2406e0e0919SSridevi Ramesh raise peltool_exception("Failed to fetch all SRCs : " + str(e)) 2416e0e0919SSridevi Ramesh return src_id 2426bf23630SSridevi Ramesh 2436bf23630SSridevi Ramesh 24420f38712SPatrick Williamsdef check_for_unexpected_src( 24520f38712SPatrick Williams unexpected_src_list=[], include_hidden_pels=False 24620f38712SPatrick Williams): 2476bf23630SSridevi Ramesh r""" 2486bf23630SSridevi Ramesh From the given unexpected SRC list, check if any unexpected SRC created 2496bf23630SSridevi Ramesh on the BMC. Returns 0 if no SRC found else throws exception. 2506bf23630SSridevi Ramesh 2516bf23630SSridevi Ramesh Description of arguments: 2526bf23630SSridevi Ramesh unexpected_src_list Give unexpected SRCs in the list format. 2536bf23630SSridevi Ramesh e.g.: ["BBXXYYYY", "AAXXYYYY"]. 254d1cb3252SSridevi Ramesh 255d1cb3252SSridevi Ramesh include_hidden_pels True/False (default: False). 256d1cb3252SSridevi Ramesh Set True to get hidden PELs else False. 2576bf23630SSridevi Ramesh """ 2586bf23630SSridevi Ramesh try: 2596bf23630SSridevi Ramesh unexpected_src_count = 0 2606bf23630SSridevi Ramesh if not unexpected_src_list: 2616bf23630SSridevi Ramesh print("Unexpected SRC list is empty.") 262d1cb3252SSridevi Ramesh src_data = fetch_all_src(include_hidden_pels) 2636bf23630SSridevi Ramesh for src in unexpected_src_list: 2646bf23630SSridevi Ramesh if src in src_data: 2656bf23630SSridevi Ramesh print("Found an unexpected SRC : " + src) 2666bf23630SSridevi Ramesh unexpected_src_count = unexpected_src_count + 1 26720f38712SPatrick Williams if unexpected_src_count >= 1: 2686bf23630SSridevi Ramesh raise peltool_exception("Unexpected SRC found.") 2696bf23630SSridevi Ramesh 2706bf23630SSridevi Ramesh except Exception as e: 27120f38712SPatrick Williams raise peltool_exception( 27220f38712SPatrick Williams "Failed to verify unexpected SRC list : " + str(e) 27320f38712SPatrick Williams ) 2746bf23630SSridevi Ramesh return unexpected_src_count 275e8801a3fSAnusha Dathatri 276e8801a3fSAnusha Dathatri 277e8801a3fSAnusha Dathatridef filter_unexpected_srcs(expected_srcs=None): 278e8801a3fSAnusha Dathatri r""" 279e8801a3fSAnusha Dathatri Return list of SRCs found in BMC after filtering expected SRCs. 280e8801a3fSAnusha Dathatri If expected_srcs is None then all SRCs found in system are returned. 281e8801a3fSAnusha Dathatri 282e8801a3fSAnusha Dathatri Description of arguments: 283e8801a3fSAnusha Dathatri expected_srcs List of expected SRCs. E.g. ["BBXXYYYY", "AAXXYYYY"]. 284e8801a3fSAnusha Dathatri """ 285e8801a3fSAnusha Dathatri 286e8801a3fSAnusha Dathatri srcs_found = fetch_all_src() 287e8801a3fSAnusha Dathatri if not expected_srcs: 288e8801a3fSAnusha Dathatri expected_srcs = [] 289e8801a3fSAnusha Dathatri print(expected_srcs) 290e8801a3fSAnusha Dathatri return list(set(srcs_found) - set(expected_srcs)) 2915636ad12SAnusha Dathatri 2925636ad12SAnusha Dathatri 2935636ad12SAnusha Dathatridef get_bmc_event_log_id_for_pel(pel_id): 2945636ad12SAnusha Dathatri r""" 2955636ad12SAnusha Dathatri Return BMC event log ID for the given PEL ID. 2965636ad12SAnusha Dathatri 2975636ad12SAnusha Dathatri Description of arguments: 2985636ad12SAnusha Dathatri pel_id PEL ID. E.g. 0x50000021. 2995636ad12SAnusha Dathatri """ 3005636ad12SAnusha Dathatri 3015636ad12SAnusha Dathatri pel_data = peltool("-i " + pel_id) 3025636ad12SAnusha Dathatri print(pel_data) 3035636ad12SAnusha Dathatri bmc_id_for_pel = pel_data["Private Header"]["BMC Event Log Id"] 3045636ad12SAnusha Dathatri return bmc_id_for_pel 305