1e7e9171eSGeorge Keishing#!/usr/bin/env python3 20bbd860fSMichael Walsh 30bbd860fSMichael Walshr""" 40bbd860fSMichael WalshThis module is the python counterpart to obmc_boot_test. 50bbd860fSMichael Walsh""" 60bbd860fSMichael Walsh 7e635ddc0SGeorge Keishingimport glob 820f38712SPatrick Williamsimport imp 920f38712SPatrick Williamsimport os 100b93fbf8SMichael Walshimport random 110ad0f7f8SMichael Walshimport re 12f566fb1fSMichael Walshimport signal 1320f38712SPatrick Williamsimport time 1420f38712SPatrick Williams 15d54bbc22SGeorge Keishingtry: 160b93fbf8SMichael Walsh import cPickle as pickle 17d54bbc22SGeorge Keishingexcept ImportError: 18d54bbc22SGeorge Keishing import pickle 1920f38712SPatrick Williams 20dc80d67dSMichael Walshimport socket 210b93fbf8SMichael Walsh 22e635ddc0SGeorge Keishingimport gen_arg as ga 23e635ddc0SGeorge Keishingimport gen_cmd as gc 2420f38712SPatrick Williamsimport gen_misc as gm 2520f38712SPatrick Williamsimport gen_plug_in_utils as gpu 2620f38712SPatrick Williamsimport gen_print as gp 27e635ddc0SGeorge Keishingimport gen_robot_keyword as grk 2820f38712SPatrick Williamsimport gen_robot_plug_in as grpi 2920f38712SPatrick Williamsimport gen_valid as gv 3020f38712SPatrick Williamsimport logging_utils as log 3120f38712SPatrick Williamsimport pel_utils as pel 3255302295SMichael Walshimport state as st 33ff340006SMichael Walshimport var_stack as vs 3420f38712SPatrick Williamsfrom boot_data import * 3520f38712SPatrick Williamsfrom robot.libraries.BuiltIn import BuiltIn 3620f38712SPatrick Williamsfrom robot.utils import DotDict 370bbd860fSMichael Walsh 3820f38712SPatrick Williamsbase_path = ( 3920f38712SPatrick Williams os.path.dirname(os.path.dirname(imp.find_module("gen_robot_print")[1])) 4020f38712SPatrick Williams + os.sep 4120f38712SPatrick Williams) 420b93fbf8SMichael Walshsys.path.append(base_path + "extended/") 4309679890SGeorge Keishingimport run_keyword as rk # NOQA 440bbd860fSMichael Walsh 45e1e26448SMichael Walsh# Setting master_pid correctly influences the behavior of plug-ins like 46e1e26448SMichael Walsh# DB_Logging 47e1e26448SMichael Walshprogram_pid = os.getpid() 4820f38712SPatrick Williamsmaster_pid = os.environ.get("AUTOBOOT_MASTER_PID", program_pid) 4920f38712SPatrick Williamspgm_name = re.sub("\\.py$", "", os.path.basename(__file__)) 50e1e26448SMichael Walsh 51b5839d00SMichael Walsh# Set up boot data structures. 52986d8aeeSMichael Walshos_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 530b93fbf8SMichael Walsh 546741f740SMichael Walshboot_lists = read_boot_lists() 55986d8aeeSMichael Walsh 56986d8aeeSMichael Walsh# The maximum number of entries that can be in the boot_history global variable. 57815b1d5bSMichael Walshmax_boot_history = 10 58986d8aeeSMichael Walshboot_history = [] 596741f740SMichael Walsh 6020f38712SPatrick Williamsstate = st.return_state_constant("default_state") 616741f740SMichael Walshcp_setup_called = 0 626741f740SMichael Walshnext_boot = "" 6320f38712SPatrick Williamsbase_tool_dir_path = ( 6420f38712SPatrick Williams os.path.normpath(os.environ.get("AUTOBOOT_BASE_TOOL_DIR_PATH", "/tmp")) 6520f38712SPatrick Williams + os.sep 6620f38712SPatrick Williams) 67b5839d00SMichael Walsh 6820f38712SPatrick Williamsffdc_dir_path = os.path.normpath(os.environ.get("FFDC_DIR_PATH", "")) + os.sep 696741f740SMichael Walshboot_success = 0 70ce90a7d1SGeorge Keishing 7120f38712SPatrick Williamsstatus_dir_path = os.environ.get( 7220f38712SPatrick Williams "STATUS_DIR_PATH", "" 7320f38712SPatrick Williams) or BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="") 746741f740SMichael Walshif status_dir_path != "": 756741f740SMichael Walsh status_dir_path = os.path.normpath(status_dir_path) + os.sep 76ce90a7d1SGeorge Keishing # For plugin expecting env gen_call_robot.py 7720f38712SPatrick Williams os.environ["STATUS_DIR_PATH"] = status_dir_path 78ce90a7d1SGeorge Keishing 7920f38712SPatrick Williamsredfish_support_trans_state = int( 8020f38712SPatrick Williams os.environ.get("REDFISH_SUPPORT_TRANS_STATE", 0) 8120f38712SPatrick Williams) or int( 8220f38712SPatrick Williams BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0) 8320f38712SPatrick Williams) 8420f38712SPatrick Williamsredfish_supported = BuiltIn().get_variable_value( 8520f38712SPatrick Williams "${REDFISH_SUPPORTED}", default=False 8620f38712SPatrick Williams) 8720f38712SPatrick Williamsredfish_rest_supported = BuiltIn().get_variable_value( 8820f38712SPatrick Williams "${REDFISH_REST_SUPPORTED}", default=False 8920f38712SPatrick Williams) 9020f38712SPatrick Williamsredfish_delete_sessions = int( 9120f38712SPatrick Williams BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1) 9220f38712SPatrick Williams) 93e58df1c8SMichael Walshif redfish_supported: 9420f38712SPatrick Williams redfish = BuiltIn().get_library_instance("redfish") 95e58df1c8SMichael Walsh default_power_on = "Redfish Power On" 96e58df1c8SMichael Walsh default_power_off = "Redfish Power Off" 97870999aaSGeorge Keishing if not redfish_support_trans_state: 98cc490b41SMichael Shepos delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 9992a54bf5SMichael Shepos delete_bmcdump_cmd = "Delete All BMC Dump" 100763902a1SGeorge Keishing default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 101e58df1c8SMichael Walsh else: 102eb1fe352SGeorge Keishing delete_errlogs_cmd = "Redfish Purge Event Log" 10392a54bf5SMichael Shepos delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 1042ef6a7dbSGeorge Keishing delete_sysdump_cmd = "Redfish Delete All System Dumps" 10520f38712SPatrick Williams default_set_power_policy = ( 10620f38712SPatrick Williams "Redfish Set Power Restore Policy AlwaysOff" 10720f38712SPatrick Williams ) 108eb1fe352SGeorge Keishingelse: 1090b93fbf8SMichael Walsh default_power_on = "REST Power On" 1100b93fbf8SMichael Walsh default_power_off = "REST Power Off" 111cc490b41SMichael Shepos delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 11292a54bf5SMichael Shepos delete_bmcdump_cmd = "Delete All BMC Dump" 113a54e06f5SGeorge Keishing default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 1146741f740SMichael Walshboot_count = 0 1150bbd860fSMichael Walsh 11685678948SMichael WalshLOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 11720f38712SPatrick WilliamsAUTOBOOT_FFDC_PREFIX = os.environ.get("AUTOBOOT_FFDC_PREFIX", "") 118986d8aeeSMichael Walshffdc_prefix = AUTOBOOT_FFDC_PREFIX 119325eb548SSunil Mboot_start_time = "" 120325eb548SSunil Mboot_end_time = "" 12120f38712SPatrick Williamssave_stack = vs.var_stack("save_stack") 12220f38712SPatrick Williamsmain_func_parm_list = ["boot_stack", "stack_mode", "quiet"] 12385678948SMichael Walsh 12485678948SMichael Walsh 12589de14a4SMichael Walshdef dump_ffdc_rc(): 12689de14a4SMichael Walsh r""" 12789de14a4SMichael Walsh Return the constant dump ffdc test return code value. 12889de14a4SMichael Walsh 12989de14a4SMichael Walsh When a plug-in call point program returns this value, it indicates that 13089de14a4SMichael Walsh this program should collect FFDC. 13189de14a4SMichael Walsh """ 13289de14a4SMichael Walsh 13389de14a4SMichael Walsh return 0x00000200 13489de14a4SMichael Walsh 13589de14a4SMichael Walsh 13689de14a4SMichael Walshdef stop_test_rc(): 13789de14a4SMichael Walsh r""" 13889de14a4SMichael Walsh Return the constant stop test return code value. 13989de14a4SMichael Walsh 14089de14a4SMichael Walsh When a plug-in call point program returns this value, it indicates that 14189de14a4SMichael Walsh this program should stop running. 14289de14a4SMichael Walsh """ 14389de14a4SMichael Walsh 14489de14a4SMichael Walsh return 0x00000200 14589de14a4SMichael Walsh 14689de14a4SMichael Walsh 14720f38712SPatrick Williamsdef process_host(host, host_var_name=""): 1480ad0f7f8SMichael Walsh r""" 1490ad0f7f8SMichael Walsh Process a host by getting the associated host name and IP address and 1500ad0f7f8SMichael Walsh setting them in global variables. 1510ad0f7f8SMichael Walsh 1520ad0f7f8SMichael Walsh If the caller does not pass the host_var_name, this function will try to 1530ad0f7f8SMichael Walsh figure out the name of the variable used by the caller for the host parm. 1540ad0f7f8SMichael Walsh Callers are advised to explicitly specify the host_var_name when calling 1550ad0f7f8SMichael Walsh with an exec command. In such cases, the get_arg_name cannot figure out 1560ad0f7f8SMichael Walsh the host variable name. 1570ad0f7f8SMichael Walsh 1580ad0f7f8SMichael Walsh This function will then create similar global variable names by 1590ad0f7f8SMichael Walsh removing "_host" and appending "_host_name" or "_ip" to the host variable 1600ad0f7f8SMichael Walsh name. 1610ad0f7f8SMichael Walsh 1620ad0f7f8SMichael Walsh Example: 1630ad0f7f8SMichael Walsh 1640ad0f7f8SMichael Walsh If a call is made like this: 1650ad0f7f8SMichael Walsh process_host(openbmc_host) 1660ad0f7f8SMichael Walsh 1670ad0f7f8SMichael Walsh Global variables openbmc_host_name and openbmc_ip will be set. 1680ad0f7f8SMichael Walsh 1690ad0f7f8SMichael Walsh Description of argument(s): 1700ad0f7f8SMichael Walsh host A host name or IP. The name of the variable used should 1710ad0f7f8SMichael Walsh have a suffix of "_host". 1720ad0f7f8SMichael Walsh host_var_name The name of the variable being used as the host parm. 1730ad0f7f8SMichael Walsh """ 1740ad0f7f8SMichael Walsh 1750ad0f7f8SMichael Walsh if host_var_name == "": 1760ad0f7f8SMichael Walsh host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 1770ad0f7f8SMichael Walsh 1780ad0f7f8SMichael Walsh host_name_var_name = re.sub("host", "host_name", host_var_name) 1790ad0f7f8SMichael Walsh ip_var_name = re.sub("host", "ip", host_var_name) 18020f38712SPatrick Williams cmd_buf = ( 18120f38712SPatrick Williams "global " 18220f38712SPatrick Williams + host_name_var_name 18320f38712SPatrick Williams + ", " 18420f38712SPatrick Williams + ip_var_name 18520f38712SPatrick Williams + " ; " 18620f38712SPatrick Williams + host_name_var_name 18720f38712SPatrick Williams + ", " 18820f38712SPatrick Williams + ip_var_name 18920f38712SPatrick Williams + " = gm.get_host_name_ip('" 19020f38712SPatrick Williams + host 19120f38712SPatrick Williams + "')" 19220f38712SPatrick Williams ) 1930ad0f7f8SMichael Walsh exec(cmd_buf) 1940ad0f7f8SMichael Walsh 1950ad0f7f8SMichael Walsh 196b5839d00SMichael Walshdef process_pgm_parms(): 197b5839d00SMichael Walsh r""" 198b5839d00SMichael Walsh Process the program parameters by assigning them all to corresponding 199b5839d00SMichael Walsh globals. Also, set some global values that depend on program parameters. 200b5839d00SMichael Walsh """ 201b5839d00SMichael Walsh 202b5839d00SMichael Walsh # Program parameter processing. 203b5839d00SMichael Walsh # Assign all program parms to python variables which are global to this 204b5839d00SMichael Walsh # module. 205b5839d00SMichael Walsh 206b5839d00SMichael Walsh global parm_list 207b5839d00SMichael Walsh parm_list = BuiltIn().get_variable_value("${parm_list}") 208b5839d00SMichael Walsh # The following subset of parms should be processed as integers. 20920f38712SPatrick Williams int_list = [ 21020f38712SPatrick Williams "max_num_tests", 21120f38712SPatrick Williams "boot_pass", 21220f38712SPatrick Williams "boot_fail", 21320f38712SPatrick Williams "ffdc_only", 21420f38712SPatrick Williams "boot_fail_threshold", 21520f38712SPatrick Williams "delete_errlogs", 21620f38712SPatrick Williams "call_post_stack_plug", 21720f38712SPatrick Williams "do_pre_boot_plug_in_setup", 21820f38712SPatrick Williams "quiet", 21920f38712SPatrick Williams "test_mode", 22020f38712SPatrick Williams "debug", 22120f38712SPatrick Williams ] 222b5839d00SMichael Walsh for parm in parm_list: 223b5839d00SMichael Walsh if parm in int_list: 22420f38712SPatrick Williams sub_cmd = ( 22520f38712SPatrick Williams 'int(BuiltIn().get_variable_value("${' + parm + '}", "0"))' 22620f38712SPatrick Williams ) 227b5839d00SMichael Walsh else: 22820f38712SPatrick Williams sub_cmd = 'BuiltIn().get_variable_value("${' + parm + '}")' 229b5839d00SMichael Walsh cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 230ff340006SMichael Walsh gp.dpissuing(cmd_buf) 231b5839d00SMichael Walsh exec(cmd_buf) 2320ad0f7f8SMichael Walsh if re.match(r".*_host$", parm): 2330ad0f7f8SMichael Walsh cmd_buf = "process_host(" + parm + ", '" + parm + "')" 2340ad0f7f8SMichael Walsh exec(cmd_buf) 2350ad0f7f8SMichael Walsh if re.match(r".*_password$", parm): 2360ad0f7f8SMichael Walsh # Register the value of any parm whose name ends in _password. 2370ad0f7f8SMichael Walsh # This will cause the print functions to replace passwords with 2380ad0f7f8SMichael Walsh # asterisks in the output. 2390ad0f7f8SMichael Walsh cmd_buf = "gp.register_passwords(" + parm + ")" 2400ad0f7f8SMichael Walsh exec(cmd_buf) 241b5839d00SMichael Walsh 242b5839d00SMichael Walsh global ffdc_dir_path_style 243b5839d00SMichael Walsh global boot_list 244b5839d00SMichael Walsh global boot_stack 245b5839d00SMichael Walsh global boot_results_file_path 246b5839d00SMichael Walsh global boot_results 247986d8aeeSMichael Walsh global boot_history 248b5839d00SMichael Walsh global ffdc_list_file_path 249e0cf8d70SMichael Walsh global ffdc_report_list_path 250600876daSMichael Walsh global ffdc_summary_list_path 251a3e7b222SMichael Walsh global boot_table 252a3e7b222SMichael Walsh global valid_boot_types 253b5839d00SMichael Walsh 254b5839d00SMichael Walsh if ffdc_dir_path_style == "": 25520f38712SPatrick Williams ffdc_dir_path_style = int(os.environ.get("FFDC_DIR_PATH_STYLE", "0")) 256b5839d00SMichael Walsh 257b5839d00SMichael Walsh # Convert these program parms to lists for easier processing.. 25836efbc04SGeorge Keishing boot_list = list(filter(None, boot_list.split(":"))) 25936efbc04SGeorge Keishing boot_stack = list(filter(None, boot_stack.split(":"))) 260b5839d00SMichael Walsh 261a3e7b222SMichael Walsh boot_table = create_boot_table(boot_table_path, os_host=os_host) 262a3e7b222SMichael Walsh valid_boot_types = create_valid_boot_list(boot_table) 263a3e7b222SMichael Walsh 264903e0b20SMichael Walsh cleanup_boot_results_file() 26520f38712SPatrick Williams boot_results_file_path = create_boot_results_file_path( 26620f38712SPatrick Williams pgm_name, openbmc_nickname, master_pid 26720f38712SPatrick Williams ) 268b5839d00SMichael Walsh 269b5839d00SMichael Walsh if os.path.isfile(boot_results_file_path): 270b5839d00SMichael Walsh # We've been called before in this run so we'll load the saved 271986d8aeeSMichael Walsh # boot_results and boot_history objects. 27220f38712SPatrick Williams boot_results, boot_history = pickle.load( 27320f38712SPatrick Williams open(boot_results_file_path, "rb") 27420f38712SPatrick Williams ) 275b5839d00SMichael Walsh else: 276b5839d00SMichael Walsh boot_results = boot_results(boot_table, boot_pass, boot_fail) 277b5839d00SMichael Walsh 27820f38712SPatrick Williams ffdc_list_file_path = ( 27920f38712SPatrick Williams base_tool_dir_path + openbmc_nickname + "/FFDC_FILE_LIST" 28020f38712SPatrick Williams ) 28120f38712SPatrick Williams ffdc_report_list_path = ( 28220f38712SPatrick Williams base_tool_dir_path + openbmc_nickname + "/FFDC_REPORT_FILE_LIST" 28320f38712SPatrick Williams ) 284b5839d00SMichael Walsh 28520f38712SPatrick Williams ffdc_summary_list_path = ( 28620f38712SPatrick Williams base_tool_dir_path + openbmc_nickname + "/FFDC_SUMMARY_FILE_LIST" 28720f38712SPatrick Williams ) 288600876daSMichael Walsh 289b5839d00SMichael Walsh 29085678948SMichael Walshdef initial_plug_in_setup(): 29185678948SMichael Walsh r""" 29285678948SMichael Walsh Initialize all plug-in environment variables which do not change for the 29385678948SMichael Walsh duration of the program. 29485678948SMichael Walsh 29585678948SMichael Walsh """ 29685678948SMichael Walsh 29785678948SMichael Walsh global LOG_LEVEL 29885678948SMichael Walsh BuiltIn().set_log_level("NONE") 29985678948SMichael Walsh 30085678948SMichael Walsh BuiltIn().set_global_variable("${master_pid}", master_pid) 30185678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 30285678948SMichael Walsh BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 30385678948SMichael Walsh BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 30420f38712SPatrick Williams BuiltIn().set_global_variable( 30520f38712SPatrick Williams "${FFDC_LIST_FILE_PATH}", ffdc_list_file_path 30620f38712SPatrick Williams ) 30720f38712SPatrick Williams BuiltIn().set_global_variable( 30820f38712SPatrick Williams "${FFDC_REPORT_LIST_PATH}", ffdc_report_list_path 30920f38712SPatrick Williams ) 31020f38712SPatrick Williams BuiltIn().set_global_variable( 31120f38712SPatrick Williams "${FFDC_SUMMARY_LIST_PATH}", ffdc_summary_list_path 31220f38712SPatrick Williams ) 31385678948SMichael Walsh 31420f38712SPatrick Williams BuiltIn().set_global_variable( 31520f38712SPatrick Williams "${FFDC_DIR_PATH_STYLE}", ffdc_dir_path_style 31620f38712SPatrick Williams ) 31720f38712SPatrick Williams BuiltIn().set_global_variable("${FFDC_CHECK}", ffdc_check) 31885678948SMichael Walsh 31985678948SMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 32085678948SMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 32185678948SMichael Walsh # element in additional_values. 32220f38712SPatrick Williams additional_values = [ 32320f38712SPatrick Williams "program_pid", 32420f38712SPatrick Williams "master_pid", 32520f38712SPatrick Williams "ffdc_dir_path", 32620f38712SPatrick Williams "status_dir_path", 32720f38712SPatrick Williams "base_tool_dir_path", 32820f38712SPatrick Williams "ffdc_list_file_path", 32920f38712SPatrick Williams "ffdc_report_list_path", 33020f38712SPatrick Williams "ffdc_summary_list_path", 33120f38712SPatrick Williams "execdir", 33220f38712SPatrick Williams "redfish_supported", 33320f38712SPatrick Williams "redfish_rest_supported", 33420f38712SPatrick Williams "redfish_support_trans_state", 33520f38712SPatrick Williams ] 33685678948SMichael Walsh 33785678948SMichael Walsh plug_in_vars = parm_list + additional_values 33885678948SMichael Walsh 33985678948SMichael Walsh for var_name in plug_in_vars: 34085678948SMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 34185678948SMichael Walsh var_name = var_name.upper() 34285678948SMichael Walsh if var_value is None: 34385678948SMichael Walsh var_value = "" 34485678948SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 34585678948SMichael Walsh 34685678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 34785678948SMichael Walsh 34868a61162SMichael Walsh # Make sure the ffdc list directory exists. 34968a61162SMichael Walsh ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 35068a61162SMichael Walsh if not os.path.exists(ffdc_list_dir_path): 35168a61162SMichael Walsh os.makedirs(ffdc_list_dir_path) 35285678948SMichael Walsh 35385678948SMichael Walsh 3540bbd860fSMichael Walshdef plug_in_setup(): 3550bbd860fSMichael Walsh r""" 35685678948SMichael Walsh Initialize all changing plug-in environment variables for use by the 35785678948SMichael Walsh plug-in programs. 3580bbd860fSMichael Walsh """ 3590bbd860fSMichael Walsh 36085678948SMichael Walsh global LOG_LEVEL 36185678948SMichael Walsh global test_really_running 36285678948SMichael Walsh 36385678948SMichael Walsh BuiltIn().set_log_level("NONE") 36485678948SMichael Walsh 3656741f740SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 3660bbd860fSMichael Walsh if boot_pass > 1: 3670bbd860fSMichael Walsh test_really_running = 1 3680bbd860fSMichael Walsh else: 3690bbd860fSMichael Walsh test_really_running = 0 3700bbd860fSMichael Walsh 37120f38712SPatrick Williams BuiltIn().set_global_variable( 37220f38712SPatrick Williams "${test_really_running}", test_really_running 37320f38712SPatrick Williams ) 3746741f740SMichael Walsh BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 3756741f740SMichael Walsh BuiltIn().set_global_variable("${boot_pass}", boot_pass) 3766741f740SMichael Walsh BuiltIn().set_global_variable("${boot_fail}", boot_fail) 3776741f740SMichael Walsh BuiltIn().set_global_variable("${boot_success}", boot_success) 3786741f740SMichael Walsh BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 379325eb548SSunil M BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 380325eb548SSunil M BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 3814c9a6453SMichael Walsh 3820bbd860fSMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 3830bbd860fSMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 3840bbd860fSMichael Walsh # element in additional_values. 38520f38712SPatrick Williams additional_values = [ 38620f38712SPatrick Williams "boot_type_desc", 38720f38712SPatrick Williams "boot_success", 38820f38712SPatrick Williams "boot_pass", 38920f38712SPatrick Williams "boot_fail", 39020f38712SPatrick Williams "test_really_running", 39120f38712SPatrick Williams "ffdc_prefix", 39220f38712SPatrick Williams "boot_start_time", 39320f38712SPatrick Williams "boot_end_time", 39420f38712SPatrick Williams ] 3950bbd860fSMichael Walsh 39685678948SMichael Walsh plug_in_vars = additional_values 3970bbd860fSMichael Walsh 3980bbd860fSMichael Walsh for var_name in plug_in_vars: 3990bbd860fSMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 4000bbd860fSMichael Walsh var_name = var_name.upper() 4010bbd860fSMichael Walsh if var_value is None: 4020bbd860fSMichael Walsh var_value = "" 4036741f740SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 4040bbd860fSMichael Walsh 4050bbd860fSMichael Walsh if debug: 40620f38712SPatrick Williams shell_rc, out_buf = gc.cmd_fnc_u( 40720f38712SPatrick Williams "printenv | egrep AUTOBOOT_ | sort -u" 40820f38712SPatrick Williams ) 4090bbd860fSMichael Walsh 41085678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 41185678948SMichael Walsh 4120bbd860fSMichael Walsh 413e0cf8d70SMichael Walshdef pre_boot_plug_in_setup(): 414e0cf8d70SMichael Walsh # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 415e0cf8d70SMichael Walsh try: 416e0cf8d70SMichael Walsh os.remove(ffdc_list_file_path) 417e0cf8d70SMichael Walsh except OSError: 418e0cf8d70SMichael Walsh pass 419e0cf8d70SMichael Walsh 420e0cf8d70SMichael Walsh # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 421e0cf8d70SMichael Walsh try: 422e0cf8d70SMichael Walsh os.remove(ffdc_report_list_path) 423e0cf8d70SMichael Walsh except OSError: 424e0cf8d70SMichael Walsh pass 425e0cf8d70SMichael Walsh 426600876daSMichael Walsh # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 427600876daSMichael Walsh try: 428600876daSMichael Walsh os.remove(ffdc_summary_list_path) 429600876daSMichael Walsh except OSError: 430600876daSMichael Walsh pass 431600876daSMichael Walsh 432e1974b96SMichael Walsh global ffdc_prefix 433e1974b96SMichael Walsh 434e1974b96SMichael Walsh seconds = time.time() 435e1974b96SMichael Walsh loc_time = time.localtime(seconds) 436e1974b96SMichael Walsh time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 437e1974b96SMichael Walsh 438e1974b96SMichael Walsh ffdc_prefix = openbmc_nickname + "." + time_string 439e1974b96SMichael Walsh 440e0cf8d70SMichael Walsh 44120f38712SPatrick Williamsdef default_sigusr1(signal_number=0, frame=None): 442f566fb1fSMichael Walsh r""" 443f566fb1fSMichael Walsh Handle SIGUSR1 by doing nothing. 444f566fb1fSMichael Walsh 445f566fb1fSMichael Walsh This function assists in debugging SIGUSR1 processing by printing messages 446f566fb1fSMichael Walsh to stdout and to the log.html file. 447f566fb1fSMichael Walsh 448f566fb1fSMichael Walsh Description of argument(s): 449f566fb1fSMichael Walsh signal_number The signal number (should always be 10 for SIGUSR1). 450f566fb1fSMichael Walsh frame The frame data. 451f566fb1fSMichael Walsh """ 452f566fb1fSMichael Walsh 45380dddde9SMichael Walsh gp.qprintn() 45480dddde9SMichael Walsh gp.qprint_executing() 455f566fb1fSMichael Walsh gp.lprint_executing() 456f566fb1fSMichael Walsh 457f566fb1fSMichael Walsh 458f566fb1fSMichael Walshdef set_default_siguser1(): 459f566fb1fSMichael Walsh r""" 460f566fb1fSMichael Walsh Set the default_sigusr1 function to be the SIGUSR1 handler. 461f566fb1fSMichael Walsh """ 462f566fb1fSMichael Walsh 46380dddde9SMichael Walsh gp.qprintn() 46480dddde9SMichael Walsh gp.qprint_executing() 465f566fb1fSMichael Walsh gp.lprint_executing() 466f566fb1fSMichael Walsh signal.signal(signal.SIGUSR1, default_sigusr1) 467f566fb1fSMichael Walsh 468f566fb1fSMichael Walsh 4696741f740SMichael Walshdef setup(): 4700bbd860fSMichael Walsh r""" 4716741f740SMichael Walsh Do general program setup tasks. 4720bbd860fSMichael Walsh """ 4730bbd860fSMichael Walsh 4746741f740SMichael Walsh global cp_setup_called 47581816748SMichael Walsh global transitional_boot_selected 4760bbd860fSMichael Walsh 477b5839d00SMichael Walsh gp.qprintn() 478b5839d00SMichael Walsh 479a54e06f5SGeorge Keishing if redfish_supported: 480a54e06f5SGeorge Keishing redfish.login() 481a54e06f5SGeorge Keishing 482f566fb1fSMichael Walsh set_default_siguser1() 48381816748SMichael Walsh transitional_boot_selected = False 48481816748SMichael Walsh 48583f4bc77SMichael Walsh robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 48683f4bc77SMichael Walsh repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 487d061c043SMichael Walsh # If we can't find process_plug_in_packages.py, ssh_pw or 488d061c043SMichael Walsh # validate_plug_ins.py, then we don't have our repo bin in PATH. 48920f38712SPatrick Williams shell_rc, out_buf = gc.cmd_fnc_u( 49020f38712SPatrick Williams "which process_plug_in_packages.py" + " ssh_pw validate_plug_ins.py", 49120f38712SPatrick Williams quiet=1, 49220f38712SPatrick Williams print_output=0, 49320f38712SPatrick Williams show_err=0, 49420f38712SPatrick Williams ) 495b5839d00SMichael Walsh if shell_rc != 0: 49620f38712SPatrick Williams os.environ["PATH"] = repo_bin_path + ":" + os.environ.get("PATH", "") 49783f4bc77SMichael Walsh # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 49883f4bc77SMichael Walsh if robot_pgm_dir_path not in sys.path: 49983f4bc77SMichael Walsh sys.path.append(robot_pgm_dir_path) 50083f4bc77SMichael Walsh PYTHONPATH = os.environ.get("PYTHONPATH", "") 50183f4bc77SMichael Walsh if PYTHONPATH == "": 50220f38712SPatrick Williams os.environ["PYTHONPATH"] = robot_pgm_dir_path 50383f4bc77SMichael Walsh else: 50420f38712SPatrick Williams os.environ["PYTHONPATH"] = robot_pgm_dir_path + ":" + PYTHONPATH 5056741f740SMichael Walsh 5066741f740SMichael Walsh validate_parms() 5076741f740SMichael Walsh 508c108e429SMichael Walsh gp.qprint_pgm_header() 5096741f740SMichael Walsh 510eae945b4SGeorge Keishing grk.run_key_u(default_set_power_policy, ignore=1) 51111cfc8c0SMichael Walsh 51285678948SMichael Walsh initial_plug_in_setup() 51385678948SMichael Walsh 5146741f740SMichael Walsh plug_in_setup() 5156741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 51620f38712SPatrick Williams call_point="setup" 51720f38712SPatrick Williams ) 5186741f740SMichael Walsh if rc != 0: 5196741f740SMichael Walsh error_message = "Plug-in setup failed.\n" 520c108e429SMichael Walsh gp.print_error_report(error_message) 5216741f740SMichael Walsh BuiltIn().fail(error_message) 5226741f740SMichael Walsh # Setting cp_setup_called lets our Teardown know that it needs to call 5236741f740SMichael Walsh # the cleanup plug-in call point. 5246741f740SMichael Walsh cp_setup_called = 1 5256741f740SMichael Walsh 5266741f740SMichael Walsh # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 5276741f740SMichael Walsh BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 52885678948SMichael Walsh # FFDC_LOG_PATH is used by "FFDC" keyword. 52985678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 5306741f740SMichael Walsh 531dc80d67dSMichael Walsh # Also printed by FFDC. 532dc80d67dSMichael Walsh global host_name 533dc80d67dSMichael Walsh global host_ip 534dc80d67dSMichael Walsh host = socket.gethostname() 535dc80d67dSMichael Walsh host_name, host_ip = gm.get_host_name_ip(host) 536dc80d67dSMichael Walsh 537986d8aeeSMichael Walsh gp.dprint_var(boot_table) 538b5839d00SMichael Walsh gp.dprint_var(boot_lists) 5390bbd860fSMichael Walsh 5400bbd860fSMichael Walsh 5416741f740SMichael Walshdef validate_parms(): 5420bbd860fSMichael Walsh r""" 5436741f740SMichael Walsh Validate all program parameters. 5440bbd860fSMichael Walsh """ 5450bbd860fSMichael Walsh 546b5839d00SMichael Walsh process_pgm_parms() 5470bbd860fSMichael Walsh 548b5839d00SMichael Walsh gp.qprintn() 549b5839d00SMichael Walsh 550b5839d00SMichael Walsh global openbmc_model 551f5ce38c3SMichael Walsh if openbmc_model == "": 55220f38712SPatrick Williams status, ret_values = grk.run_key_u("Get BMC System Model", ignore=1) 553ec2836d6SGeorge Keishing # Set the model to default "OPENBMC" if getting it from BMC fails. 55420f38712SPatrick Williams if status == "FAIL": 55520f38712SPatrick Williams openbmc_model = "OPENBMC" 556ec2836d6SGeorge Keishing else: 557f5ce38c3SMichael Walsh openbmc_model = ret_values 558f5ce38c3SMichael Walsh BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 559f5ce38c3SMichael Walsh gv.set_exit_on_error(True) 56044cef258SMichael Walsh gv.valid_value(openbmc_host) 56144cef258SMichael Walsh gv.valid_value(openbmc_username) 56244cef258SMichael Walsh gv.valid_value(openbmc_password) 56344cef258SMichael Walsh gv.valid_value(rest_username) 56444cef258SMichael Walsh gv.valid_value(rest_password) 56544cef258SMichael Walsh gv.valid_value(ipmi_username) 56644cef258SMichael Walsh gv.valid_value(ipmi_password) 5676741f740SMichael Walsh if os_host != "": 56844cef258SMichael Walsh gv.valid_value(os_username) 56944cef258SMichael Walsh gv.valid_value(os_password) 5706741f740SMichael Walsh if pdu_host != "": 57144cef258SMichael Walsh gv.valid_value(pdu_username) 57244cef258SMichael Walsh gv.valid_value(pdu_password) 57344cef258SMichael Walsh gv.valid_integer(pdu_slot_no) 5746741f740SMichael Walsh if openbmc_serial_host != "": 57544cef258SMichael Walsh gv.valid_integer(openbmc_serial_port) 57644cef258SMichael Walsh gv.valid_value(openbmc_model) 57744cef258SMichael Walsh gv.valid_integer(max_num_tests) 57844cef258SMichael Walsh gv.valid_integer(boot_pass) 57944cef258SMichael Walsh gv.valid_integer(boot_fail) 5806741f740SMichael Walsh plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 58120f38712SPatrick Williams BuiltIn().set_global_variable( 58220f38712SPatrick Williams "${plug_in_packages_list}", plug_in_packages_list 58320f38712SPatrick Williams ) 58420f38712SPatrick Williams gv.valid_value(stack_mode, valid_values=["normal", "skip"]) 585f5ce38c3SMichael Walsh gv.set_exit_on_error(False) 586a20da401SMichael Walsh if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 58720f38712SPatrick Williams error_message = ( 58820f38712SPatrick Williams "You must provide either a value for either the" 58920f38712SPatrick Williams + " boot_list or the boot_stack parm.\n" 59020f38712SPatrick Williams ) 5916741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 5926741f740SMichael Walsh valid_boot_list(boot_list, valid_boot_types) 5936741f740SMichael Walsh valid_boot_list(boot_stack, valid_boot_types) 59420f38712SPatrick Williams selected_PDU_boots = list( 59520f38712SPatrick Williams set(boot_list + boot_stack) & set(boot_lists["PDU_reboot"]) 59620f38712SPatrick Williams ) 59711cfc8c0SMichael Walsh if len(selected_PDU_boots) > 0 and pdu_host == "": 59820f38712SPatrick Williams error_message = ( 59920f38712SPatrick Williams "You have selected the following boots which" 60020f38712SPatrick Williams + " require a PDU host but no value for pdu_host:\n" 60120f38712SPatrick Williams ) 60211cfc8c0SMichael Walsh error_message += gp.sprint_var(selected_PDU_boots) 603986d8aeeSMichael Walsh error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 60411cfc8c0SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 60511cfc8c0SMichael Walsh 6066741f740SMichael Walsh return 6070bbd860fSMichael Walsh 6080bbd860fSMichael Walsh 6096741f740SMichael Walshdef my_get_state(): 6100bbd860fSMichael Walsh r""" 6116741f740SMichael Walsh Get the system state plus a little bit of wrapping. 6120bbd860fSMichael Walsh """ 6130bbd860fSMichael Walsh 6146741f740SMichael Walsh global state 6156741f740SMichael Walsh 61620f38712SPatrick Williams req_states = ["epoch_seconds"] + st.default_req_states 6176741f740SMichael Walsh 618b5839d00SMichael Walsh gp.qprint_timen("Getting system state.") 6196741f740SMichael Walsh if test_mode: 62020f38712SPatrick Williams state["epoch_seconds"] = int(time.time()) 6216741f740SMichael Walsh else: 622b5839d00SMichael Walsh state = st.get_state(req_states=req_states, quiet=quiet) 623b5839d00SMichael Walsh gp.qprint_var(state) 624341c21ebSMichael Walsh 625341c21ebSMichael Walsh 62645ca6e4cSMichael Walshdef valid_state(): 62745ca6e4cSMichael Walsh r""" 62845ca6e4cSMichael Walsh Verify that our state dictionary contains no blank values. If we don't get 62945ca6e4cSMichael Walsh valid state data, we cannot continue to work. 63045ca6e4cSMichael Walsh """ 63145ca6e4cSMichael Walsh 63220f38712SPatrick Williams if st.compare_states(state, st.invalid_state_match, "or"): 63320f38712SPatrick Williams error_message = ( 63420f38712SPatrick Williams "The state dictionary contains blank fields which" 63520f38712SPatrick Williams + " is illegal.\n" 63620f38712SPatrick Williams + gp.sprint_var(state) 63720f38712SPatrick Williams ) 63845ca6e4cSMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 63945ca6e4cSMichael Walsh 64045ca6e4cSMichael Walsh 6416741f740SMichael Walshdef select_boot(): 642341c21ebSMichael Walsh r""" 643341c21ebSMichael Walsh Select a boot test to be run based on our current state and return the 644341c21ebSMichael Walsh chosen boot type. 645341c21ebSMichael Walsh 646341c21ebSMichael Walsh Description of arguments: 6476741f740SMichael Walsh state The state of the machine. 648341c21ebSMichael Walsh """ 649341c21ebSMichael Walsh 65081816748SMichael Walsh global transitional_boot_selected 65130dadae2SMichael Walsh global boot_stack 65230dadae2SMichael Walsh 653b5839d00SMichael Walsh gp.qprint_timen("Selecting a boot test.") 6546741f740SMichael Walsh 65581816748SMichael Walsh if transitional_boot_selected and not boot_success: 65681816748SMichael Walsh prior_boot = next_boot 65781816748SMichael Walsh boot_candidate = boot_stack.pop() 65820f38712SPatrick Williams gp.qprint_timen( 65920f38712SPatrick Williams "The prior '" 66020f38712SPatrick Williams + next_boot 66120f38712SPatrick Williams + "' was chosen to" 66220f38712SPatrick Williams + " transition to a valid state for '" 66320f38712SPatrick Williams + boot_candidate 664004ad3c9SJoy Onyerikwu + "' which was at the top of the boot_stack. Since" 66520f38712SPatrick Williams + " the '" 66620f38712SPatrick Williams + next_boot 66720f38712SPatrick Williams + "' failed, the '" 66820f38712SPatrick Williams + boot_candidate 66920f38712SPatrick Williams + "' has been removed from the stack" 67020f38712SPatrick Williams + " to avoid and endless failure loop." 67120f38712SPatrick Williams ) 67281816748SMichael Walsh if len(boot_stack) == 0: 67381816748SMichael Walsh return "" 67481816748SMichael Walsh 6756741f740SMichael Walsh my_get_state() 67645ca6e4cSMichael Walsh valid_state() 6776741f740SMichael Walsh 67881816748SMichael Walsh transitional_boot_selected = False 6796741f740SMichael Walsh stack_popped = 0 6806741f740SMichael Walsh if len(boot_stack) > 0: 6816741f740SMichael Walsh stack_popped = 1 682b5839d00SMichael Walsh gp.qprint_dashes() 683b5839d00SMichael Walsh gp.qprint_var(boot_stack) 684b5839d00SMichael Walsh gp.qprint_dashes() 685b5839d00SMichael Walsh skip_boot_printed = 0 686b5839d00SMichael Walsh while len(boot_stack) > 0: 6876741f740SMichael Walsh boot_candidate = boot_stack.pop() 68820f38712SPatrick Williams if stack_mode == "normal": 689b5839d00SMichael Walsh break 690b5839d00SMichael Walsh else: 69120f38712SPatrick Williams if st.compare_states(state, boot_table[boot_candidate]["end"]): 692b5839d00SMichael Walsh if not skip_boot_printed: 693ff340006SMichael Walsh gp.qprint_var(stack_mode) 694ff340006SMichael Walsh gp.qprintn() 69520f38712SPatrick Williams gp.qprint_timen( 69620f38712SPatrick Williams "Skipping the following boot tests" 697004ad3c9SJoy Onyerikwu + " which are unnecessary since their" 698004ad3c9SJoy Onyerikwu + " required end states match the" 69920f38712SPatrick Williams + " current machine state:" 70020f38712SPatrick Williams ) 701b5839d00SMichael Walsh skip_boot_printed = 1 702ff340006SMichael Walsh gp.qprint_var(boot_candidate) 703b5839d00SMichael Walsh boot_candidate = "" 704b5839d00SMichael Walsh if boot_candidate == "": 705b5839d00SMichael Walsh gp.qprint_dashes() 706b5839d00SMichael Walsh gp.qprint_var(boot_stack) 707b5839d00SMichael Walsh gp.qprint_dashes() 708b5839d00SMichael Walsh return boot_candidate 70920f38712SPatrick Williams if st.compare_states(state, boot_table[boot_candidate]["start"]): 71020f38712SPatrick Williams gp.qprint_timen( 71120f38712SPatrick Williams "The machine state is valid for a '" 71220f38712SPatrick Williams + boot_candidate 71320f38712SPatrick Williams + "' boot test." 71420f38712SPatrick Williams ) 715b5839d00SMichael Walsh gp.qprint_dashes() 716b5839d00SMichael Walsh gp.qprint_var(boot_stack) 717b5839d00SMichael Walsh gp.qprint_dashes() 7186741f740SMichael Walsh return boot_candidate 719341c21ebSMichael Walsh else: 72020f38712SPatrick Williams gp.qprint_timen( 72120f38712SPatrick Williams "The machine state does not match the required" 72220f38712SPatrick Williams + " starting state for a '" 72320f38712SPatrick Williams + boot_candidate 72420f38712SPatrick Williams + "' boot test:" 72520f38712SPatrick Williams ) 72620f38712SPatrick Williams gp.qprint_varx( 72720f38712SPatrick Williams "boot_table_start_entry", boot_table[boot_candidate]["start"] 72820f38712SPatrick Williams ) 7296741f740SMichael Walsh boot_stack.append(boot_candidate) 73081816748SMichael Walsh transitional_boot_selected = True 7316741f740SMichael Walsh popped_boot = boot_candidate 7326741f740SMichael Walsh 7336741f740SMichael Walsh # Loop through your list selecting a boot_candidates 7346741f740SMichael Walsh boot_candidates = [] 7356741f740SMichael Walsh for boot_candidate in boot_list: 73620f38712SPatrick Williams if st.compare_states(state, boot_table[boot_candidate]["start"]): 7376741f740SMichael Walsh if stack_popped: 73820f38712SPatrick Williams if st.compare_states( 73920f38712SPatrick Williams boot_table[boot_candidate]["end"], 74020f38712SPatrick Williams boot_table[popped_boot]["start"], 74120f38712SPatrick Williams ): 7426741f740SMichael Walsh boot_candidates.append(boot_candidate) 743341c21ebSMichael Walsh else: 7446741f740SMichael Walsh boot_candidates.append(boot_candidate) 7456741f740SMichael Walsh 7466741f740SMichael Walsh if len(boot_candidates) == 0: 74720f38712SPatrick Williams gp.qprint_timen( 74820f38712SPatrick Williams "The user's boot list contained no boot tests" 74920f38712SPatrick Williams + " which are valid for the current machine state." 75020f38712SPatrick Williams ) 7516741f740SMichael Walsh boot_candidate = default_power_on 75220f38712SPatrick Williams if not st.compare_states(state, boot_table[default_power_on]["start"]): 7536741f740SMichael Walsh boot_candidate = default_power_off 7546741f740SMichael Walsh boot_candidates.append(boot_candidate) 75520f38712SPatrick Williams gp.qprint_timen( 75620f38712SPatrick Williams "Using default '" 75720f38712SPatrick Williams + boot_candidate 75820f38712SPatrick Williams + "' boot type to transition to valid state." 75920f38712SPatrick Williams ) 7606741f740SMichael Walsh 761b5839d00SMichael Walsh gp.dprint_var(boot_candidates) 7626741f740SMichael Walsh 7636741f740SMichael Walsh # Randomly select a boot from the candidate list. 7646741f740SMichael Walsh boot = random.choice(boot_candidates) 765341c21ebSMichael Walsh 766341c21ebSMichael Walsh return boot 7670bbd860fSMichael Walsh 76855302295SMichael Walsh 769b2e53ecdSMichael Walshdef print_defect_report(ffdc_file_list): 770341c21ebSMichael Walsh r""" 771341c21ebSMichael Walsh Print a defect report. 772b2e53ecdSMichael Walsh 773b2e53ecdSMichael Walsh Description of argument(s): 774b2e53ecdSMichael Walsh ffdc_file_list A list of files which were collected by our ffdc functions. 775341c21ebSMichael Walsh """ 776341c21ebSMichael Walsh 777600876daSMichael Walsh # Making deliberate choice to NOT run plug_in_setup(). We don't want 778600876daSMichael Walsh # ffdc_prefix updated. 779600876daSMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 78020f38712SPatrick Williams call_point="ffdc_report", stop_on_plug_in_failure=0 78120f38712SPatrick Williams ) 782600876daSMichael Walsh 783e0cf8d70SMichael Walsh # Get additional header data which may have been created by ffdc plug-ins. 784e0cf8d70SMichael Walsh # Also, delete the individual header files to cleanup. 78520f38712SPatrick Williams cmd_buf = ( 78620f38712SPatrick Williams "file_list=$(cat " 78720f38712SPatrick Williams + ffdc_report_list_path 78820f38712SPatrick Williams + " 2>/dev/null)" 78920f38712SPatrick Williams + ' ; [ ! -z "${file_list}" ] && cat ${file_list}' 79020f38712SPatrick Williams + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 79120f38712SPatrick Williams ) 79220f38712SPatrick Williams shell_rc, more_header_info = gc.cmd_fnc_u( 79320f38712SPatrick Williams cmd_buf, print_output=0, show_err=0 79420f38712SPatrick Williams ) 795e0cf8d70SMichael Walsh 796b2e53ecdSMichael Walsh # Get additional summary data which may have been created by ffdc plug-ins. 797600876daSMichael Walsh # Also, delete the individual header files to cleanup. 79820f38712SPatrick Williams cmd_buf = ( 79920f38712SPatrick Williams "file_list=$(cat " 80020f38712SPatrick Williams + ffdc_summary_list_path 80120f38712SPatrick Williams + " 2>/dev/null)" 80220f38712SPatrick Williams + ' ; [ ! -z "${file_list}" ] && cat ${file_list}' 80320f38712SPatrick Williams + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 80420f38712SPatrick Williams ) 80520f38712SPatrick Williams shell_rc, ffdc_summary_info = gc.cmd_fnc_u( 80620f38712SPatrick Williams cmd_buf, print_output=0, show_err=0 80720f38712SPatrick Williams ) 808600876daSMichael Walsh 809b2e53ecdSMichael Walsh # ffdc_list_file_path contains a list of any ffdc files created by plug- 810b2e53ecdSMichael Walsh # ins, etc. Read that data into a list. 811341c21ebSMichael Walsh try: 81220f38712SPatrick Williams plug_in_ffdc_list = ( 81320f38712SPatrick Williams open(ffdc_list_file_path, "r").read().rstrip("\n").split("\n") 81420f38712SPatrick Williams ) 81536efbc04SGeorge Keishing plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 816341c21ebSMichael Walsh except IOError: 817b2e53ecdSMichael Walsh plug_in_ffdc_list = [] 818b2e53ecdSMichael Walsh 819b2e53ecdSMichael Walsh # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 820b2e53ecdSMichael Walsh # in. Eliminate duplicates and sort the list. 821004ad3c9SJoy Onyerikwu ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 822b2e53ecdSMichael Walsh 823b2e53ecdSMichael Walsh if status_file_path != "": 824b2e53ecdSMichael Walsh ffdc_file_list.insert(0, status_file_path) 825b2e53ecdSMichael Walsh 826b2e53ecdSMichael Walsh # Convert the list to a printable list. 827b2e53ecdSMichael Walsh printable_ffdc_file_list = "\n".join(ffdc_file_list) 828341c21ebSMichael Walsh 82968a61162SMichael Walsh # Open ffdc_file_list for writing. We will write a complete list of 83068a61162SMichael Walsh # FFDC files to it for possible use by plug-ins like cp_stop_check. 83120f38712SPatrick Williams ffdc_list_file = open(ffdc_list_file_path, "w") 832b2e53ecdSMichael Walsh ffdc_list_file.write(printable_ffdc_file_list + "\n") 833b2e53ecdSMichael Walsh ffdc_list_file.close() 834b2e53ecdSMichael Walsh 835b2e53ecdSMichael Walsh indent = 0 836b2e53ecdSMichael Walsh width = 90 837b2e53ecdSMichael Walsh linefeed = 1 838b2e53ecdSMichael Walsh char = "=" 83968a61162SMichael Walsh 84068a61162SMichael Walsh gp.qprintn() 841b2e53ecdSMichael Walsh gp.qprint_dashes(indent, width, linefeed, char) 84268a61162SMichael Walsh gp.qprintn("Copy this data to the defect:\n") 84368a61162SMichael Walsh 844e0cf8d70SMichael Walsh if len(more_header_info) > 0: 845ff340006SMichael Walsh gp.qprintn(more_header_info) 84620f38712SPatrick Williams gp.qpvars( 84720f38712SPatrick Williams host_name, 84820f38712SPatrick Williams host_ip, 84920f38712SPatrick Williams openbmc_nickname, 85020f38712SPatrick Williams openbmc_host, 85120f38712SPatrick Williams openbmc_host_name, 85220f38712SPatrick Williams openbmc_ip, 85320f38712SPatrick Williams openbmc_username, 85420f38712SPatrick Williams openbmc_password, 85520f38712SPatrick Williams rest_username, 85620f38712SPatrick Williams rest_password, 85720f38712SPatrick Williams ipmi_username, 85820f38712SPatrick Williams ipmi_password, 85920f38712SPatrick Williams os_host, 86020f38712SPatrick Williams os_host_name, 86120f38712SPatrick Williams os_ip, 86220f38712SPatrick Williams os_username, 86320f38712SPatrick Williams os_password, 86420f38712SPatrick Williams pdu_host, 86520f38712SPatrick Williams pdu_host_name, 86620f38712SPatrick Williams pdu_ip, 86720f38712SPatrick Williams pdu_username, 86820f38712SPatrick Williams pdu_password, 86920f38712SPatrick Williams pdu_slot_no, 87020f38712SPatrick Williams openbmc_serial_host, 87120f38712SPatrick Williams openbmc_serial_host_name, 87220f38712SPatrick Williams openbmc_serial_ip, 87320f38712SPatrick Williams openbmc_serial_port, 87420f38712SPatrick Williams ) 87568a61162SMichael Walsh 87668a61162SMichael Walsh gp.qprintn() 877986d8aeeSMichael Walsh print_boot_history(boot_history) 87868a61162SMichael Walsh gp.qprintn() 87968a61162SMichael Walsh gp.qprint_var(state) 880b5839d00SMichael Walsh gp.qprintn() 881b5839d00SMichael Walsh gp.qprintn("FFDC data files:") 882b2e53ecdSMichael Walsh gp.qprintn(printable_ffdc_file_list) 883b5839d00SMichael Walsh gp.qprintn() 884341c21ebSMichael Walsh 885600876daSMichael Walsh if len(ffdc_summary_info) > 0: 886ff340006SMichael Walsh gp.qprintn(ffdc_summary_info) 887600876daSMichael Walsh 888b2e53ecdSMichael Walsh gp.qprint_dashes(indent, width, linefeed, char) 88968a61162SMichael Walsh 8906741f740SMichael Walsh 8916741f740SMichael Walshdef my_ffdc(): 8926741f740SMichael Walsh r""" 8936741f740SMichael Walsh Collect FFDC data. 8946741f740SMichael Walsh """ 8956741f740SMichael Walsh 8966741f740SMichael Walsh global state 8976741f740SMichael Walsh 8986741f740SMichael Walsh plug_in_setup() 8996741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 90020f38712SPatrick Williams call_point="ffdc", stop_on_plug_in_failure=0 90120f38712SPatrick Williams ) 9026741f740SMichael Walsh 90320f38712SPatrick Williams AUTOBOOT_FFDC_PREFIX = os.environ["AUTOBOOT_FFDC_PREFIX"] 90420f38712SPatrick Williams status, ffdc_file_list = grk.run_key_u( 90520f38712SPatrick Williams "FFDC ffdc_prefix=" 906004ad3c9SJoy Onyerikwu + AUTOBOOT_FFDC_PREFIX 907004ad3c9SJoy Onyerikwu + " ffdc_function_list=" 90820f38712SPatrick Williams + ffdc_function_list, 90920f38712SPatrick Williams ignore=1, 91020f38712SPatrick Williams ) 91120f38712SPatrick Williams if status != "PASS": 912ff340006SMichael Walsh gp.qprint_error("Call to ffdc failed.\n") 913c9bd2e87SMichael Walsh if type(ffdc_file_list) is not list: 914c9bd2e87SMichael Walsh ffdc_file_list = [] 915c9bd2e87SMichael Walsh # Leave a record for caller that "soft" errors occurred. 916c9bd2e87SMichael Walsh soft_errors = 1 917c9bd2e87SMichael Walsh gpu.save_plug_in_value(soft_errors, pgm_name) 9186741f740SMichael Walsh 9196741f740SMichael Walsh my_get_state() 9206741f740SMichael Walsh 921b2e53ecdSMichael Walsh print_defect_report(ffdc_file_list) 9226741f740SMichael Walsh 9236741f740SMichael Walsh 9246741f740SMichael Walshdef print_test_start_message(boot_keyword): 9256741f740SMichael Walsh r""" 9266741f740SMichael Walsh Print a message indicating what boot test is about to run. 9276741f740SMichael Walsh 9286741f740SMichael Walsh Description of arguments: 9296741f740SMichael Walsh boot_keyword The name of the boot which is to be run 9306741f740SMichael Walsh (e.g. "BMC Power On"). 9316741f740SMichael Walsh """ 9326741f740SMichael Walsh 933986d8aeeSMichael Walsh global boot_history 934325eb548SSunil M global boot_start_time 9356741f740SMichael Walsh 93620f38712SPatrick Williams doing_msg = gp.sprint_timen('Doing "' + boot_keyword + '".') 937325eb548SSunil M 938325eb548SSunil M # Set boot_start_time for use by plug-ins. 939325eb548SSunil M boot_start_time = doing_msg[1:33] 940325eb548SSunil M gp.qprint_var(boot_start_time) 941325eb548SSunil M 942b5839d00SMichael Walsh gp.qprint(doing_msg) 9436741f740SMichael Walsh 944986d8aeeSMichael Walsh update_boot_history(boot_history, doing_msg, max_boot_history) 9456741f740SMichael Walsh 9466741f740SMichael Walsh 94720f38712SPatrick Williamsdef stop_boot_test(signal_number=0, frame=None): 948f566fb1fSMichael Walsh r""" 949f566fb1fSMichael Walsh Handle SIGUSR1 by aborting the boot test that is running. 950f566fb1fSMichael Walsh 951f566fb1fSMichael Walsh Description of argument(s): 952f566fb1fSMichael Walsh signal_number The signal number (should always be 10 for SIGUSR1). 953f566fb1fSMichael Walsh frame The frame data. 954f566fb1fSMichael Walsh """ 955f566fb1fSMichael Walsh 95680dddde9SMichael Walsh gp.qprintn() 95780dddde9SMichael Walsh gp.qprint_executing() 958f566fb1fSMichael Walsh gp.lprint_executing() 959f566fb1fSMichael Walsh 960f566fb1fSMichael Walsh # Restore original sigusr1 handler. 961f566fb1fSMichael Walsh set_default_siguser1() 962f566fb1fSMichael Walsh 963f566fb1fSMichael Walsh message = "The caller has asked that the boot test be stopped and marked" 964f566fb1fSMichael Walsh message += " as a failure." 965f566fb1fSMichael Walsh 966f566fb1fSMichael Walsh function_stack = gm.get_function_stack() 967f566fb1fSMichael Walsh if "wait_state" in function_stack: 968c44aa538SMichael Walsh st.set_exit_wait_early_message(message) 969f566fb1fSMichael Walsh else: 970f566fb1fSMichael Walsh BuiltIn().fail(gp.sprint_error(message)) 971f566fb1fSMichael Walsh 972f566fb1fSMichael Walsh 9736741f740SMichael Walshdef run_boot(boot): 9746741f740SMichael Walsh r""" 9756741f740SMichael Walsh Run the specified boot. 9766741f740SMichael Walsh 9776741f740SMichael Walsh Description of arguments: 9786741f740SMichael Walsh boot The name of the boot test to be performed. 9796741f740SMichael Walsh """ 9806741f740SMichael Walsh 9816741f740SMichael Walsh global state 9826741f740SMichael Walsh 983f566fb1fSMichael Walsh signal.signal(signal.SIGUSR1, stop_boot_test) 984f566fb1fSMichael Walsh gp.qprint_timen("stop_boot_test is armed.") 985f566fb1fSMichael Walsh 9866741f740SMichael Walsh print_test_start_message(boot) 9876741f740SMichael Walsh 9886741f740SMichael Walsh plug_in_setup() 98920f38712SPatrick Williams rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 99020f38712SPatrick Williams call_point="pre_boot" 99120f38712SPatrick Williams ) 9926741f740SMichael Walsh if rc != 0: 99320f38712SPatrick Williams error_message = ( 99420f38712SPatrick Williams "Plug-in failed with non-zero return code.\n" 99520f38712SPatrick Williams + gp.sprint_var(rc, fmt=gp.hexa()) 99620f38712SPatrick Williams ) 997f566fb1fSMichael Walsh set_default_siguser1() 9986741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 9996741f740SMichael Walsh 10006741f740SMichael Walsh if test_mode: 10016741f740SMichael Walsh # In test mode, we'll pretend the boot worked by assigning its 10026741f740SMichael Walsh # required end state to the default state value. 100320f38712SPatrick Williams state = st.strip_anchor_state(boot_table[boot]["end"]) 10046741f740SMichael Walsh else: 10056741f740SMichael Walsh # Assertion: We trust that the state data was made fresh by the 10066741f740SMichael Walsh # caller. 10076741f740SMichael Walsh 1008b5839d00SMichael Walsh gp.qprintn() 10096741f740SMichael Walsh 101020f38712SPatrick Williams if boot_table[boot]["method_type"] == "keyword": 101120f38712SPatrick Williams rk.my_run_keywords( 101220f38712SPatrick Williams boot_table[boot].get("lib_file_path", ""), 101320f38712SPatrick Williams boot_table[boot]["method"], 101420f38712SPatrick Williams quiet=quiet, 101520f38712SPatrick Williams ) 10166741f740SMichael Walsh 101720f38712SPatrick Williams if boot_table[boot]["bmc_reboot"]: 101820f38712SPatrick Williams st.wait_for_comm_cycle(int(state["epoch_seconds"])) 101930dadae2SMichael Walsh plug_in_setup() 102020f38712SPatrick Williams rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 102120f38712SPatrick Williams call_point="post_reboot" 102220f38712SPatrick Williams ) 102330dadae2SMichael Walsh if rc != 0: 10240b93fbf8SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" 1025986d8aeeSMichael Walsh error_message += gp.sprint_var(rc, fmt=gp.hexa()) 1026f566fb1fSMichael Walsh set_default_siguser1() 102730dadae2SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 10286741f740SMichael Walsh else: 10296741f740SMichael Walsh match_state = st.anchor_state(state) 103020f38712SPatrick Williams del match_state["epoch_seconds"] 10316741f740SMichael Walsh # Wait for the state to change in any way. 103220f38712SPatrick Williams st.wait_state( 103320f38712SPatrick Williams match_state, 103420f38712SPatrick Williams wait_time=state_change_timeout, 103520f38712SPatrick Williams interval="10 seconds", 103620f38712SPatrick Williams invert=1, 103720f38712SPatrick Williams ) 10386741f740SMichael Walsh 1039b5839d00SMichael Walsh gp.qprintn() 104020f38712SPatrick Williams if boot_table[boot]["end"]["chassis"] == "Off": 10416741f740SMichael Walsh boot_timeout = power_off_timeout 10426741f740SMichael Walsh else: 10436741f740SMichael Walsh boot_timeout = power_on_timeout 104420f38712SPatrick Williams st.wait_state( 104520f38712SPatrick Williams boot_table[boot]["end"], 104620f38712SPatrick Williams wait_time=boot_timeout, 104720f38712SPatrick Williams interval="10 seconds", 104820f38712SPatrick Williams ) 10496741f740SMichael Walsh 10506741f740SMichael Walsh plug_in_setup() 105120f38712SPatrick Williams rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 105220f38712SPatrick Williams call_point="post_boot" 105320f38712SPatrick Williams ) 10546741f740SMichael Walsh if rc != 0: 105520f38712SPatrick Williams error_message = ( 105620f38712SPatrick Williams "Plug-in failed with non-zero return code.\n" 105720f38712SPatrick Williams + gp.sprint_var(rc, fmt=gp.hexa()) 105820f38712SPatrick Williams ) 1059f566fb1fSMichael Walsh set_default_siguser1() 10606741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 10616741f740SMichael Walsh 1062f566fb1fSMichael Walsh # Restore original sigusr1 handler. 1063f566fb1fSMichael Walsh set_default_siguser1() 1064f566fb1fSMichael Walsh 10656741f740SMichael Walsh 10666741f740SMichael Walshdef test_loop_body(): 10676741f740SMichael Walsh r""" 10686741f740SMichael Walsh The main loop body for the loop in main_py. 10696741f740SMichael Walsh 10706741f740SMichael Walsh Description of arguments: 10716741f740SMichael Walsh boot_count The iteration number (starts at 1). 10726741f740SMichael Walsh """ 10736741f740SMichael Walsh 10746741f740SMichael Walsh global boot_count 10756741f740SMichael Walsh global state 10766741f740SMichael Walsh global next_boot 10776741f740SMichael Walsh global boot_success 1078325eb548SSunil M global boot_end_time 10796741f740SMichael Walsh 1080*02d32765SYi Hu # The flag can be enabled or disabled on the go 1081*02d32765SYi Hu redfish_delete_sessions = int( 1082*02d32765SYi Hu BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1) 1083*02d32765SYi Hu ) 1084*02d32765SYi Hu 1085b5839d00SMichael Walsh gp.qprintn() 10866741f740SMichael Walsh 10876741f740SMichael Walsh next_boot = select_boot() 1088b5839d00SMichael Walsh if next_boot == "": 1089b5839d00SMichael Walsh return True 10906741f740SMichael Walsh 1091b5839d00SMichael Walsh boot_count += 1 1092b5839d00SMichael Walsh gp.qprint_timen("Starting boot " + str(boot_count) + ".") 10936741f740SMichael Walsh 1094e0cf8d70SMichael Walsh pre_boot_plug_in_setup() 10956741f740SMichael Walsh 10966741f740SMichael Walsh cmd_buf = ["run_boot", next_boot] 10976741f740SMichael Walsh boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 10986741f740SMichael Walsh if boot_status == "FAIL": 1099b5839d00SMichael Walsh gp.qprint(msg) 11006741f740SMichael Walsh 1101b5839d00SMichael Walsh gp.qprintn() 11026741f740SMichael Walsh if boot_status == "PASS": 11036741f740SMichael Walsh boot_success = 1 110420f38712SPatrick Williams completion_msg = gp.sprint_timen( 110520f38712SPatrick Williams 'BOOT_SUCCESS: "' + next_boot + '" succeeded.' 110620f38712SPatrick Williams ) 11076741f740SMichael Walsh else: 11086741f740SMichael Walsh boot_success = 0 110920f38712SPatrick Williams completion_msg = gp.sprint_timen( 111020f38712SPatrick Williams 'BOOT_FAILED: "' + next_boot + '" failed.' 111120f38712SPatrick Williams ) 1112325eb548SSunil M 1113325eb548SSunil M # Set boot_end_time for use by plug-ins. 1114325eb548SSunil M boot_end_time = completion_msg[1:33] 1115325eb548SSunil M gp.qprint_var(boot_end_time) 1116325eb548SSunil M 1117325eb548SSunil M gp.qprint(completion_msg) 11186741f740SMichael Walsh 11196741f740SMichael Walsh boot_results.update(next_boot, boot_status) 11206741f740SMichael Walsh 11216741f740SMichael Walsh plug_in_setup() 11226741f740SMichael Walsh # NOTE: A post_test_case call point failure is NOT counted as a boot 11236741f740SMichael Walsh # failure. 11246741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 112520f38712SPatrick Williams call_point="post_test_case", stop_on_plug_in_failure=0 112620f38712SPatrick Williams ) 11276741f740SMichael Walsh 11286741f740SMichael Walsh plug_in_setup() 11296741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 113020f38712SPatrick Williams call_point="ffdc_check", 113120f38712SPatrick Williams shell_rc=dump_ffdc_rc(), 113220f38712SPatrick Williams stop_on_plug_in_failure=1, 113320f38712SPatrick Williams stop_on_non_zero_rc=1, 113420f38712SPatrick Williams ) 113520f38712SPatrick Williams if ffdc_check == "All" or shell_rc == dump_ffdc_rc(): 113683f4bc77SMichael Walsh status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 113720f38712SPatrick Williams if status != "PASS": 1138ff340006SMichael Walsh gp.qprint_error("Call to my_ffdc failed.\n") 1139c9bd2e87SMichael Walsh # Leave a record for caller that "soft" errors occurred. 1140c9bd2e87SMichael Walsh soft_errors = 1 1141c9bd2e87SMichael Walsh gpu.save_plug_in_value(soft_errors, pgm_name) 11426741f740SMichael Walsh 1143aabef1e3SMichael Walsh if delete_errlogs: 11441a67b08aSMichael Shepos # print error logs before delete 1145438fd3b6SGeorge Keishing if redfish_support_trans_state: 1146438fd3b6SGeorge Keishing status, error_logs = grk.run_key_u("Get Redfish Event Logs") 114720f38712SPatrick Williams log.print_error_logs( 114820f38712SPatrick Williams error_logs, "AdditionalDataURI Message Severity" 114920f38712SPatrick Williams ) 1150438fd3b6SGeorge Keishing else: 11511a67b08aSMichael Shepos status, error_logs = grk.run_key_u("Get Error Logs") 11520e5f1137SMichael Shepos log.print_error_logs(error_logs, "AdditionalData Message Severity") 1153438fd3b6SGeorge Keishing pels = pel.peltool("-l", ignore_err=1) 11540e5f1137SMichael Shepos gp.qprint_var(pels) 11551a67b08aSMichael Shepos 1156d139f286SMichael Walsh # We need to purge error logs between boots or they build up. 1157409ad35aSMichael Walsh grk.run_key(delete_errlogs_cmd, ignore=1) 115892a54bf5SMichael Shepos grk.run_key(delete_bmcdump_cmd, ignore=1) 11592ef6a7dbSGeorge Keishing if redfish_support_trans_state: 11602ef6a7dbSGeorge Keishing grk.run_key(delete_sysdump_cmd, ignore=1) 1161d139f286SMichael Walsh 1162952f9b09SMichael Walsh boot_results.print_report() 1163b5839d00SMichael Walsh gp.qprint_timen("Finished boot " + str(boot_count) + ".") 1164952f9b09SMichael Walsh 11656741f740SMichael Walsh plug_in_setup() 11666741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 116720f38712SPatrick Williams call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1 116820f38712SPatrick Williams ) 116989de14a4SMichael Walsh if shell_rc == stop_test_rc(): 11703ba8ecdcSMichael Walsh message = "Stopping as requested by user.\n" 117180dddde9SMichael Walsh gp.qprint_time(message) 11723ba8ecdcSMichael Walsh BuiltIn().fail(message) 11736741f740SMichael Walsh 1174d139f286SMichael Walsh # This should help prevent ConnectionErrors. 11754d65c86dSGeorge Keishing # Purge all redfish and REST connection sessions. 1176d86e45c6SGeorge Keishing if redfish_delete_sessions: 11774d65c86dSGeorge Keishing grk.run_key_u("Close All Connections", ignore=1) 11784d65c86dSGeorge Keishing grk.run_key_u("Delete All Redfish Sessions", ignore=1) 1179d139f286SMichael Walsh 11806741f740SMichael Walsh return True 11816741f740SMichael Walsh 11826741f740SMichael Walsh 118383f4bc77SMichael Walshdef obmc_boot_test_teardown(): 11846741f740SMichael Walsh r""" 1185f75d4354SMichael Walsh Clean up after the main keyword. 11866741f740SMichael Walsh """ 1187f75d4354SMichael Walsh gp.qprint_executing() 1188f75d4354SMichael Walsh 1189f75d4354SMichael Walsh if ga.psutil_imported: 1190f75d4354SMichael Walsh ga.terminate_descendants() 11916741f740SMichael Walsh 11926741f740SMichael Walsh if cp_setup_called: 11936741f740SMichael Walsh plug_in_setup() 11946741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 119520f38712SPatrick Williams call_point="cleanup", stop_on_plug_in_failure=0 119620f38712SPatrick Williams ) 11976741f740SMichael Walsh 119820f38712SPatrick Williams if "boot_results_file_path" in globals(): 1199986d8aeeSMichael Walsh # Save boot_results and boot_history objects to a file in case they are 12006c64574bSMichael Walsh # needed again. 1201b5839d00SMichael Walsh gp.qprint_timen("Saving boot_results to the following path.") 1202b5839d00SMichael Walsh gp.qprint_var(boot_results_file_path) 120320f38712SPatrick Williams pickle.dump( 120420f38712SPatrick Williams (boot_results, boot_history), 120520f38712SPatrick Williams open(boot_results_file_path, "wb"), 120620f38712SPatrick Williams pickle.HIGHEST_PROTOCOL, 120720f38712SPatrick Williams ) 12080b93fbf8SMichael Walsh 1209ff340006SMichael Walsh global save_stack 1210ff340006SMichael Walsh # Restore any global values saved on the save_stack. 1211ff340006SMichael Walsh for parm_name in main_func_parm_list: 1212ff340006SMichael Walsh # Get the parm_value if it was saved on the stack. 1213ff340006SMichael Walsh try: 1214ff340006SMichael Walsh parm_value = save_stack.pop(parm_name) 1215004ad3c9SJoy Onyerikwu except BaseException: 1216ff340006SMichael Walsh # If it was not saved, no further action is required. 1217ff340006SMichael Walsh continue 1218ff340006SMichael Walsh 1219ff340006SMichael Walsh # Restore the saved value. 122020f38712SPatrick Williams cmd_buf = ( 122120f38712SPatrick Williams 'BuiltIn().set_global_variable("${' + parm_name + '}", parm_value)' 122220f38712SPatrick Williams ) 1223ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1224ff340006SMichael Walsh exec(cmd_buf) 1225ff340006SMichael Walsh 1226ff340006SMichael Walsh gp.dprintn(save_stack.sprint_obj()) 1227ff340006SMichael Walsh 12286741f740SMichael Walsh 1229c9116811SMichael Walshdef test_teardown(): 1230c9116811SMichael Walsh r""" 1231c9116811SMichael Walsh Clean up after this test case. 1232c9116811SMichael Walsh """ 1233c9116811SMichael Walsh 1234c9116811SMichael Walsh gp.qprintn() 1235f75d4354SMichael Walsh gp.qprint_executing() 1236f75d4354SMichael Walsh 1237f75d4354SMichael Walsh if ga.psutil_imported: 1238f75d4354SMichael Walsh ga.terminate_descendants() 1239f75d4354SMichael Walsh 124020f38712SPatrick Williams cmd_buf = [ 124120f38712SPatrick Williams "Print Error", 124220f38712SPatrick Williams "A keyword timeout occurred ending this program.\n", 124320f38712SPatrick Williams ] 1244c9116811SMichael Walsh BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1245c9116811SMichael Walsh 1246a54e06f5SGeorge Keishing if redfish_supported: 1247a54e06f5SGeorge Keishing redfish.logout() 1248a54e06f5SGeorge Keishing 1249c108e429SMichael Walsh gp.qprint_pgm_footer() 1250b5839d00SMichael Walsh 1251c9116811SMichael Walsh 125289de14a4SMichael Walshdef post_stack(): 125389de14a4SMichael Walsh r""" 125489de14a4SMichael Walsh Process post_stack plug-in programs. 125589de14a4SMichael Walsh """ 125689de14a4SMichael Walsh 125789de14a4SMichael Walsh if not call_post_stack_plug: 125889de14a4SMichael Walsh # The caller does not wish to have post_stack plug-in processing done. 125989de14a4SMichael Walsh return 126089de14a4SMichael Walsh 126189de14a4SMichael Walsh global boot_success 126289de14a4SMichael Walsh 126389de14a4SMichael Walsh # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 126489de14a4SMichael Walsh pre_boot_plug_in_setup() 126589de14a4SMichael Walsh # For the purposes of the following plug-ins, mark the "boot" as a success. 126689de14a4SMichael Walsh boot_success = 1 126789de14a4SMichael Walsh plug_in_setup() 126820f38712SPatrick Williams ( 126920f38712SPatrick Williams rc, 127020f38712SPatrick Williams shell_rc, 127120f38712SPatrick Williams failed_plug_in_name, 127220f38712SPatrick Williams history, 127320f38712SPatrick Williams ) = grpi.rprocess_plug_in_packages( 127420f38712SPatrick Williams call_point="post_stack", stop_on_plug_in_failure=0, return_history=True 127520f38712SPatrick Williams ) 1276986d8aeeSMichael Walsh for doing_msg in history: 1277986d8aeeSMichael Walsh update_boot_history(boot_history, doing_msg, max_boot_history) 1278815b1d5bSMichael Walsh if rc != 0: 1279815b1d5bSMichael Walsh boot_success = 0 128089de14a4SMichael Walsh 128189de14a4SMichael Walsh plug_in_setup() 128220f38712SPatrick Williams rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 128320f38712SPatrick Williams call_point="ffdc_check", 1284815b1d5bSMichael Walsh shell_rc=dump_ffdc_rc(), 1285815b1d5bSMichael Walsh stop_on_plug_in_failure=1, 128620f38712SPatrick Williams stop_on_non_zero_rc=1, 128720f38712SPatrick Williams ) 1288815b1d5bSMichael Walsh if shell_rc == dump_ffdc_rc(): 128989de14a4SMichael Walsh status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 129020f38712SPatrick Williams if status != "PASS": 129189de14a4SMichael Walsh gp.qprint_error("Call to my_ffdc failed.\n") 1292c9bd2e87SMichael Walsh # Leave a record for caller that "soft" errors occurred. 1293c9bd2e87SMichael Walsh soft_errors = 1 1294c9bd2e87SMichael Walsh gpu.save_plug_in_value(soft_errors, pgm_name) 129589de14a4SMichael Walsh 129689de14a4SMichael Walsh plug_in_setup() 129789de14a4SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 129820f38712SPatrick Williams call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1 129920f38712SPatrick Williams ) 130089de14a4SMichael Walsh if shell_rc == stop_test_rc(): 130189de14a4SMichael Walsh message = "Stopping as requested by user.\n" 130280dddde9SMichael Walsh gp.qprint_time(message) 130389de14a4SMichael Walsh BuiltIn().fail(message) 130489de14a4SMichael Walsh 130589de14a4SMichael Walsh 130620f38712SPatrick Williamsdef obmc_boot_test_py( 130720f38712SPatrick Williams loc_boot_stack=None, loc_stack_mode=None, loc_quiet=None 130820f38712SPatrick Williams): 13096741f740SMichael Walsh r""" 13106741f740SMichael Walsh Do main program processing. 13116741f740SMichael Walsh """ 13126741f740SMichael Walsh 1313ff340006SMichael Walsh global save_stack 1314ff340006SMichael Walsh 131520f38712SPatrick Williams ga.set_term_options( 131620f38712SPatrick Williams term_requests={"pgm_names": ["process_plug_in_packages.py"]} 131720f38712SPatrick Williams ) 1318f75d4354SMichael Walsh 131936efbc04SGeorge Keishing gp.dprintn() 1320ff340006SMichael Walsh # Process function parms. 1321ff340006SMichael Walsh for parm_name in main_func_parm_list: 1322ff340006SMichael Walsh # Get parm's value. 132336efbc04SGeorge Keishing parm_value = eval("loc_" + parm_name) 132436efbc04SGeorge Keishing gp.dpvars(parm_name, parm_value) 1325ff340006SMichael Walsh 132636efbc04SGeorge Keishing if parm_value is not None: 1327ff340006SMichael Walsh # Save the global value on a stack. 132820f38712SPatrick Williams cmd_buf = ( 132920f38712SPatrick Williams 'save_stack.push(BuiltIn().get_variable_value("${' 133020f38712SPatrick Williams + parm_name 133120f38712SPatrick Williams + '}"), "' 133220f38712SPatrick Williams + parm_name 133320f38712SPatrick Williams + '")' 133420f38712SPatrick Williams ) 1335ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1336ff340006SMichael Walsh exec(cmd_buf) 1337ff340006SMichael Walsh 1338ff340006SMichael Walsh # Set the global value to the passed value. 133920f38712SPatrick Williams cmd_buf = ( 134020f38712SPatrick Williams 'BuiltIn().set_global_variable("${' 134120f38712SPatrick Williams + parm_name 134220f38712SPatrick Williams + '}", loc_' 134320f38712SPatrick Williams + parm_name 134420f38712SPatrick Williams + ")" 134520f38712SPatrick Williams ) 1346ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1347ff340006SMichael Walsh exec(cmd_buf) 1348ff340006SMichael Walsh 1349ff340006SMichael Walsh gp.dprintn(save_stack.sprint_obj()) 1350b5839d00SMichael Walsh 13516741f740SMichael Walsh setup() 13526741f740SMichael Walsh 1353cd9fbfd7SMichael Walsh init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1354cd9fbfd7SMichael Walsh 1355a20da401SMichael Walsh if ffdc_only: 1356a20da401SMichael Walsh gp.qprint_timen("Caller requested ffdc_only.") 1357986d8aeeSMichael Walsh if do_pre_boot_plug_in_setup: 1358e0cf8d70SMichael Walsh pre_boot_plug_in_setup() 135983f4bc77SMichael Walsh grk.run_key_u("my_ffdc") 1360764d2f83SMichael Walsh return 1361a20da401SMichael Walsh 1362409ad35aSMichael Walsh if delete_errlogs: 13631a67b08aSMichael Shepos # print error logs before delete 1364438fd3b6SGeorge Keishing if redfish_support_trans_state: 1365438fd3b6SGeorge Keishing status, error_logs = grk.run_key_u("Get Redfish Event Logs") 136620f38712SPatrick Williams log.print_error_logs( 136720f38712SPatrick Williams error_logs, "AdditionalDataURI Message Severity" 136820f38712SPatrick Williams ) 1369438fd3b6SGeorge Keishing else: 13701a67b08aSMichael Shepos status, error_logs = grk.run_key_u("Get Error Logs") 13710e5f1137SMichael Shepos log.print_error_logs(error_logs, "AdditionalData Message Severity") 1372438fd3b6SGeorge Keishing pels = pel.peltool("-l", ignore_err=1) 13730e5f1137SMichael Shepos gp.qprint_var(pels) 13741a67b08aSMichael Shepos 1375409ad35aSMichael Walsh # Delete errlogs prior to doing any boot tests. 1376409ad35aSMichael Walsh grk.run_key(delete_errlogs_cmd, ignore=1) 137792a54bf5SMichael Shepos grk.run_key(delete_bmcdump_cmd, ignore=1) 13782ef6a7dbSGeorge Keishing if redfish_support_trans_state: 13792ef6a7dbSGeorge Keishing grk.run_key(delete_sysdump_cmd, ignore=1) 1380409ad35aSMichael Walsh 13816741f740SMichael Walsh # Process caller's boot_stack. 138220f38712SPatrick Williams while len(boot_stack) > 0: 13836741f740SMichael Walsh test_loop_body() 13846741f740SMichael Walsh 1385b5839d00SMichael Walsh gp.qprint_timen("Finished processing stack.") 138630dadae2SMichael Walsh 138789de14a4SMichael Walsh post_stack() 138889de14a4SMichael Walsh 13896741f740SMichael Walsh # Process caller's boot_list. 13906741f740SMichael Walsh if len(boot_list) > 0: 13916741f740SMichael Walsh for ix in range(1, max_num_tests + 1): 13926741f740SMichael Walsh test_loop_body() 13936741f740SMichael Walsh 1394b5839d00SMichael Walsh gp.qprint_timen("Completed all requested boot tests.") 1395b5839d00SMichael Walsh 1396b5839d00SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 1397cd9fbfd7SMichael Walsh new_fail = boot_fail - init_boot_fail 1398cd9fbfd7SMichael Walsh if new_fail > boot_fail_threshold: 139920f38712SPatrick Williams error_message = ( 140020f38712SPatrick Williams "Boot failures exceed the boot failure" 140120f38712SPatrick Williams + " threshold:\n" 140220f38712SPatrick Williams + gp.sprint_var(new_fail) 140320f38712SPatrick Williams + gp.sprint_var(boot_fail_threshold) 140420f38712SPatrick Williams ) 1405b5839d00SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 1406