10bbd860fSMichael Walsh#!/usr/bin/env python 20bbd860fSMichael Walsh 30bbd860fSMichael Walshr""" 40bbd860fSMichael WalshThis module is the python counterpart to obmc_boot_test. 50bbd860fSMichael Walsh""" 60bbd860fSMichael Walsh 70b93fbf8SMichael Walshimport os 80b93fbf8SMichael Walshimport imp 90b93fbf8SMichael Walshimport time 100b93fbf8SMichael Walshimport glob 110b93fbf8SMichael Walshimport random 120ad0f7f8SMichael Walshimport re 13f566fb1fSMichael Walshimport signal 14d54bbc22SGeorge Keishingtry: 150b93fbf8SMichael Walsh import cPickle as pickle 16d54bbc22SGeorge Keishingexcept ImportError: 17d54bbc22SGeorge Keishing import pickle 18dc80d67dSMichael Walshimport socket 190b93fbf8SMichael Walsh 200b93fbf8SMichael Walshfrom robot.utils import DotDict 210b93fbf8SMichael Walshfrom robot.libraries.BuiltIn import BuiltIn 220b93fbf8SMichael Walsh 236741f740SMichael Walshfrom boot_data import * 24c9116811SMichael Walshimport gen_print as gp 2555302295SMichael Walshimport gen_robot_plug_in as grpi 266741f740SMichael Walshimport gen_robot_valid as grv 276741f740SMichael Walshimport gen_misc as gm 286741f740SMichael Walshimport gen_cmd as gc 29b5839d00SMichael Walshimport gen_robot_keyword as grk 3055302295SMichael Walshimport state as st 31ff340006SMichael Walshimport var_stack as vs 32c9bd2e87SMichael Walshimport gen_plug_in_utils as gpu 330bbd860fSMichael Walsh 340b93fbf8SMichael Walshbase_path = os.path.dirname(os.path.dirname( 350b93fbf8SMichael Walsh imp.find_module("gen_robot_print")[1])) +\ 360b93fbf8SMichael Walsh os.sep 370b93fbf8SMichael Walshsys.path.append(base_path + "extended/") 380b93fbf8SMichael Walshimport run_keyword as rk 390bbd860fSMichael Walsh 40e1e26448SMichael Walsh# Setting master_pid correctly influences the behavior of plug-ins like 41e1e26448SMichael Walsh# DB_Logging 42e1e26448SMichael Walshprogram_pid = os.getpid() 43e1e26448SMichael Walshmaster_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 44004ad3c9SJoy Onyerikwupgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 45e1e26448SMichael Walsh 46b5839d00SMichael Walsh# Set up boot data structures. 47*986d8aeeSMichael Walshos_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 48*986d8aeeSMichael Walshboot_table = create_boot_table(os_host=os_host) 49b5839d00SMichael Walshvalid_boot_types = create_valid_boot_list(boot_table) 500b93fbf8SMichael Walsh 516741f740SMichael Walshboot_lists = read_boot_lists() 52*986d8aeeSMichael Walsh 53*986d8aeeSMichael Walsh# The maximum number of entries that can be in the boot_history global variable. 54815b1d5bSMichael Walshmax_boot_history = 10 55*986d8aeeSMichael Walshboot_history = [] 566741f740SMichael Walsh 577dc885b6SMichael Walshstate = st.return_state_constant('default_state') 586741f740SMichael Walshcp_setup_called = 0 596741f740SMichael Walshnext_boot = "" 606741f740SMichael Walshbase_tool_dir_path = os.path.normpath(os.environ.get( 616741f740SMichael Walsh 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 62b5839d00SMichael Walsh 636741f740SMichael Walshffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 646741f740SMichael Walshboot_success = 0 656741f740SMichael Walshstatus_dir_path = os.environ.get('STATUS_DIR_PATH', "") 666741f740SMichael Walshif status_dir_path != "": 676741f740SMichael Walsh status_dir_path = os.path.normpath(status_dir_path) + os.sep 680b93fbf8SMichael Walshdefault_power_on = "REST Power On" 690b93fbf8SMichael Walshdefault_power_off = "REST Power Off" 706741f740SMichael Walshboot_count = 0 710bbd860fSMichael Walsh 7285678948SMichael WalshLOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 73*986d8aeeSMichael WalshAUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 74*986d8aeeSMichael Walshffdc_prefix = AUTOBOOT_FFDC_PREFIX 75325eb548SSunil Mboot_start_time = "" 76325eb548SSunil Mboot_end_time = "" 77ff340006SMichael Walshsave_stack = vs.var_stack('save_stack') 78ff340006SMichael Walshmain_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 7985678948SMichael Walsh 8085678948SMichael Walsh 8189de14a4SMichael Walshdef dump_ffdc_rc(): 8289de14a4SMichael Walsh r""" 8389de14a4SMichael Walsh Return the constant dump ffdc test return code value. 8489de14a4SMichael Walsh 8589de14a4SMichael Walsh When a plug-in call point program returns this value, it indicates that 8689de14a4SMichael Walsh this program should collect FFDC. 8789de14a4SMichael Walsh """ 8889de14a4SMichael Walsh 8989de14a4SMichael Walsh return 0x00000200 9089de14a4SMichael Walsh 9189de14a4SMichael Walsh 9289de14a4SMichael Walshdef stop_test_rc(): 9389de14a4SMichael Walsh r""" 9489de14a4SMichael Walsh Return the constant stop test return code value. 9589de14a4SMichael Walsh 9689de14a4SMichael Walsh When a plug-in call point program returns this value, it indicates that 9789de14a4SMichael Walsh this program should stop running. 9889de14a4SMichael Walsh """ 9989de14a4SMichael Walsh 10089de14a4SMichael Walsh return 0x00000200 10189de14a4SMichael Walsh 10289de14a4SMichael Walsh 1030ad0f7f8SMichael Walshdef process_host(host, 1040ad0f7f8SMichael Walsh host_var_name=""): 1050ad0f7f8SMichael Walsh r""" 1060ad0f7f8SMichael Walsh Process a host by getting the associated host name and IP address and 1070ad0f7f8SMichael Walsh setting them in global variables. 1080ad0f7f8SMichael Walsh 1090ad0f7f8SMichael Walsh If the caller does not pass the host_var_name, this function will try to 1100ad0f7f8SMichael Walsh figure out the name of the variable used by the caller for the host parm. 1110ad0f7f8SMichael Walsh Callers are advised to explicitly specify the host_var_name when calling 1120ad0f7f8SMichael Walsh with an exec command. In such cases, the get_arg_name cannot figure out 1130ad0f7f8SMichael Walsh the host variable name. 1140ad0f7f8SMichael Walsh 1150ad0f7f8SMichael Walsh This function will then create similar global variable names by 1160ad0f7f8SMichael Walsh removing "_host" and appending "_host_name" or "_ip" to the host variable 1170ad0f7f8SMichael Walsh name. 1180ad0f7f8SMichael Walsh 1190ad0f7f8SMichael Walsh Example: 1200ad0f7f8SMichael Walsh 1210ad0f7f8SMichael Walsh If a call is made like this: 1220ad0f7f8SMichael Walsh process_host(openbmc_host) 1230ad0f7f8SMichael Walsh 1240ad0f7f8SMichael Walsh Global variables openbmc_host_name and openbmc_ip will be set. 1250ad0f7f8SMichael Walsh 1260ad0f7f8SMichael Walsh Description of argument(s): 1270ad0f7f8SMichael Walsh host A host name or IP. The name of the variable used should 1280ad0f7f8SMichael Walsh have a suffix of "_host". 1290ad0f7f8SMichael Walsh host_var_name The name of the variable being used as the host parm. 1300ad0f7f8SMichael Walsh """ 1310ad0f7f8SMichael Walsh 1320ad0f7f8SMichael Walsh if host_var_name == "": 1330ad0f7f8SMichael Walsh host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 1340ad0f7f8SMichael Walsh 1350ad0f7f8SMichael Walsh host_name_var_name = re.sub("host", "host_name", host_var_name) 1360ad0f7f8SMichael Walsh ip_var_name = re.sub("host", "ip", host_var_name) 1370ad0f7f8SMichael Walsh cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 1380ad0f7f8SMichael Walsh host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 1390ad0f7f8SMichael Walsh host + "')" 1400ad0f7f8SMichael Walsh exec(cmd_buf) 1410ad0f7f8SMichael Walsh 1420ad0f7f8SMichael Walsh 143b5839d00SMichael Walshdef process_pgm_parms(): 144b5839d00SMichael Walsh r""" 145b5839d00SMichael Walsh Process the program parameters by assigning them all to corresponding 146b5839d00SMichael Walsh globals. Also, set some global values that depend on program parameters. 147b5839d00SMichael Walsh """ 148b5839d00SMichael Walsh 149b5839d00SMichael Walsh # Program parameter processing. 150b5839d00SMichael Walsh # Assign all program parms to python variables which are global to this 151b5839d00SMichael Walsh # module. 152b5839d00SMichael Walsh 153b5839d00SMichael Walsh global parm_list 154b5839d00SMichael Walsh parm_list = BuiltIn().get_variable_value("${parm_list}") 155b5839d00SMichael Walsh # The following subset of parms should be processed as integers. 156b5839d00SMichael Walsh int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 15789de14a4SMichael Walsh 'boot_fail_threshold', 'delete_errlogs', 158*986d8aeeSMichael Walsh 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 159*986d8aeeSMichael Walsh 'test_mode', 'debug'] 160b5839d00SMichael Walsh for parm in parm_list: 161b5839d00SMichael Walsh if parm in int_list: 162b5839d00SMichael Walsh sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 163b5839d00SMichael Walsh "}\", \"0\"))" 164b5839d00SMichael Walsh else: 165b5839d00SMichael Walsh sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 166b5839d00SMichael Walsh cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 167ff340006SMichael Walsh gp.dpissuing(cmd_buf) 168b5839d00SMichael Walsh exec(cmd_buf) 1690ad0f7f8SMichael Walsh if re.match(r".*_host$", parm): 1700ad0f7f8SMichael Walsh cmd_buf = "process_host(" + parm + ", '" + parm + "')" 1710ad0f7f8SMichael Walsh exec(cmd_buf) 1720ad0f7f8SMichael Walsh if re.match(r".*_password$", parm): 1730ad0f7f8SMichael Walsh # Register the value of any parm whose name ends in _password. 1740ad0f7f8SMichael Walsh # This will cause the print functions to replace passwords with 1750ad0f7f8SMichael Walsh # asterisks in the output. 1760ad0f7f8SMichael Walsh cmd_buf = "gp.register_passwords(" + parm + ")" 1770ad0f7f8SMichael Walsh exec(cmd_buf) 178b5839d00SMichael Walsh 179b5839d00SMichael Walsh global ffdc_dir_path_style 180b5839d00SMichael Walsh global boot_list 181b5839d00SMichael Walsh global boot_stack 182b5839d00SMichael Walsh global boot_results_file_path 183b5839d00SMichael Walsh global boot_results 184*986d8aeeSMichael Walsh global boot_history 185b5839d00SMichael Walsh global ffdc_list_file_path 186e0cf8d70SMichael Walsh global ffdc_report_list_path 187600876daSMichael Walsh global ffdc_summary_list_path 188b5839d00SMichael Walsh 189b5839d00SMichael Walsh if ffdc_dir_path_style == "": 190b5839d00SMichael Walsh ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 191b5839d00SMichael Walsh 192b5839d00SMichael Walsh # Convert these program parms to lists for easier processing.. 19336efbc04SGeorge Keishing boot_list = list(filter(None, boot_list.split(":"))) 19436efbc04SGeorge Keishing boot_stack = list(filter(None, boot_stack.split(":"))) 195b5839d00SMichael Walsh 196903e0b20SMichael Walsh cleanup_boot_results_file() 197903e0b20SMichael Walsh boot_results_file_path = create_boot_results_file_path(pgm_name, 198903e0b20SMichael Walsh openbmc_nickname, 199903e0b20SMichael Walsh master_pid) 200b5839d00SMichael Walsh 201b5839d00SMichael Walsh if os.path.isfile(boot_results_file_path): 202b5839d00SMichael Walsh # We've been called before in this run so we'll load the saved 203*986d8aeeSMichael Walsh # boot_results and boot_history objects. 204*986d8aeeSMichael Walsh boot_results, boot_history =\ 2056c64574bSMichael Walsh pickle.load(open(boot_results_file_path, 'rb')) 206b5839d00SMichael Walsh else: 207b5839d00SMichael Walsh boot_results = boot_results(boot_table, boot_pass, boot_fail) 208b5839d00SMichael Walsh 209b5839d00SMichael Walsh ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 210b5839d00SMichael Walsh "/FFDC_FILE_LIST" 211e0cf8d70SMichael Walsh ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 212e0cf8d70SMichael Walsh "/FFDC_REPORT_FILE_LIST" 213b5839d00SMichael Walsh 214600876daSMichael Walsh ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 215600876daSMichael Walsh "/FFDC_SUMMARY_FILE_LIST" 216600876daSMichael Walsh 217b5839d00SMichael Walsh 21885678948SMichael Walshdef initial_plug_in_setup(): 21985678948SMichael Walsh r""" 22085678948SMichael Walsh Initialize all plug-in environment variables which do not change for the 22185678948SMichael Walsh duration of the program. 22285678948SMichael Walsh 22385678948SMichael Walsh """ 22485678948SMichael Walsh 22585678948SMichael Walsh global LOG_LEVEL 22685678948SMichael Walsh BuiltIn().set_log_level("NONE") 22785678948SMichael Walsh 22885678948SMichael Walsh BuiltIn().set_global_variable("${master_pid}", master_pid) 22985678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 23085678948SMichael Walsh BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 23185678948SMichael Walsh BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 23285678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 23385678948SMichael Walsh ffdc_list_file_path) 234e0cf8d70SMichael Walsh BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 235e0cf8d70SMichael Walsh ffdc_report_list_path) 236600876daSMichael Walsh BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 237600876daSMichael Walsh ffdc_summary_list_path) 23885678948SMichael Walsh 23985678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 24085678948SMichael Walsh ffdc_dir_path_style) 24185678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_CHECK}", 24285678948SMichael Walsh ffdc_check) 24385678948SMichael Walsh 24485678948SMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 24585678948SMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 24685678948SMichael Walsh # element in additional_values. 24785678948SMichael Walsh additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 24885678948SMichael Walsh "status_dir_path", "base_tool_dir_path", 249600876daSMichael Walsh "ffdc_list_file_path", "ffdc_report_list_path", 2500a3bdb4cSMichael Walsh "ffdc_summary_list_path", "execdir"] 25185678948SMichael Walsh 25285678948SMichael Walsh plug_in_vars = parm_list + additional_values 25385678948SMichael Walsh 25485678948SMichael Walsh for var_name in plug_in_vars: 25585678948SMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 25685678948SMichael Walsh var_name = var_name.upper() 25785678948SMichael Walsh if var_value is None: 25885678948SMichael Walsh var_value = "" 25985678948SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 26085678948SMichael Walsh 26185678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 26285678948SMichael Walsh 26368a61162SMichael Walsh # Make sure the ffdc list directory exists. 26468a61162SMichael Walsh ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 26568a61162SMichael Walsh if not os.path.exists(ffdc_list_dir_path): 26668a61162SMichael Walsh os.makedirs(ffdc_list_dir_path) 26785678948SMichael Walsh 26885678948SMichael Walsh 2690bbd860fSMichael Walshdef plug_in_setup(): 2700bbd860fSMichael Walsh r""" 27185678948SMichael Walsh Initialize all changing plug-in environment variables for use by the 27285678948SMichael Walsh plug-in programs. 2730bbd860fSMichael Walsh """ 2740bbd860fSMichael Walsh 27585678948SMichael Walsh global LOG_LEVEL 27685678948SMichael Walsh global test_really_running 27785678948SMichael Walsh 27885678948SMichael Walsh BuiltIn().set_log_level("NONE") 27985678948SMichael Walsh 2806741f740SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 2810bbd860fSMichael Walsh if boot_pass > 1: 2820bbd860fSMichael Walsh test_really_running = 1 2830bbd860fSMichael Walsh else: 2840bbd860fSMichael Walsh test_really_running = 0 2850bbd860fSMichael Walsh 2866741f740SMichael Walsh BuiltIn().set_global_variable("${test_really_running}", 2876741f740SMichael Walsh test_really_running) 2886741f740SMichael Walsh BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 2896741f740SMichael Walsh BuiltIn().set_global_variable("${boot_pass}", boot_pass) 2906741f740SMichael Walsh BuiltIn().set_global_variable("${boot_fail}", boot_fail) 2916741f740SMichael Walsh BuiltIn().set_global_variable("${boot_success}", boot_success) 2926741f740SMichael Walsh BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 293325eb548SSunil M BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 294325eb548SSunil M BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 2954c9a6453SMichael Walsh 2960bbd860fSMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 2970bbd860fSMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 2980bbd860fSMichael Walsh # element in additional_values. 2990bbd860fSMichael Walsh additional_values = ["boot_type_desc", "boot_success", "boot_pass", 300325eb548SSunil M "boot_fail", "test_really_running", "ffdc_prefix", 301325eb548SSunil M "boot_start_time", "boot_end_time"] 3020bbd860fSMichael Walsh 30385678948SMichael Walsh plug_in_vars = additional_values 3040bbd860fSMichael Walsh 3050bbd860fSMichael Walsh for var_name in plug_in_vars: 3060bbd860fSMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 3070bbd860fSMichael Walsh var_name = var_name.upper() 3080bbd860fSMichael Walsh if var_value is None: 3090bbd860fSMichael Walsh var_value = "" 3106741f740SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 3110bbd860fSMichael Walsh 3120bbd860fSMichael Walsh if debug: 3136741f740SMichael Walsh shell_rc, out_buf = \ 3146741f740SMichael Walsh gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 3150bbd860fSMichael Walsh 31685678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 31785678948SMichael Walsh 3180bbd860fSMichael Walsh 319e0cf8d70SMichael Walshdef pre_boot_plug_in_setup(): 320e0cf8d70SMichael Walsh 321e0cf8d70SMichael Walsh # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 322e0cf8d70SMichael Walsh try: 323e0cf8d70SMichael Walsh os.remove(ffdc_list_file_path) 324e0cf8d70SMichael Walsh except OSError: 325e0cf8d70SMichael Walsh pass 326e0cf8d70SMichael Walsh 327e0cf8d70SMichael Walsh # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 328e0cf8d70SMichael Walsh try: 329e0cf8d70SMichael Walsh os.remove(ffdc_report_list_path) 330e0cf8d70SMichael Walsh except OSError: 331e0cf8d70SMichael Walsh pass 332e0cf8d70SMichael Walsh 333600876daSMichael Walsh # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 334600876daSMichael Walsh try: 335600876daSMichael Walsh os.remove(ffdc_summary_list_path) 336600876daSMichael Walsh except OSError: 337600876daSMichael Walsh pass 338600876daSMichael Walsh 339e1974b96SMichael Walsh global ffdc_prefix 340e1974b96SMichael Walsh 341e1974b96SMichael Walsh seconds = time.time() 342e1974b96SMichael Walsh loc_time = time.localtime(seconds) 343e1974b96SMichael Walsh time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 344e1974b96SMichael Walsh 345e1974b96SMichael Walsh ffdc_prefix = openbmc_nickname + "." + time_string 346e1974b96SMichael Walsh 347e0cf8d70SMichael Walsh 348f566fb1fSMichael Walshdef default_sigusr1(signal_number=0, 349f566fb1fSMichael Walsh frame=None): 350f566fb1fSMichael Walsh r""" 351f566fb1fSMichael Walsh Handle SIGUSR1 by doing nothing. 352f566fb1fSMichael Walsh 353f566fb1fSMichael Walsh This function assists in debugging SIGUSR1 processing by printing messages 354f566fb1fSMichael Walsh to stdout and to the log.html file. 355f566fb1fSMichael Walsh 356f566fb1fSMichael Walsh Description of argument(s): 357f566fb1fSMichael Walsh signal_number The signal number (should always be 10 for SIGUSR1). 358f566fb1fSMichael Walsh frame The frame data. 359f566fb1fSMichael Walsh """ 360f566fb1fSMichael Walsh 361f566fb1fSMichael Walsh gp.printn() 362f566fb1fSMichael Walsh gp.print_executing() 363f566fb1fSMichael Walsh gp.lprint_executing() 364f566fb1fSMichael Walsh 365f566fb1fSMichael Walsh 366f566fb1fSMichael Walshdef set_default_siguser1(): 367f566fb1fSMichael Walsh r""" 368f566fb1fSMichael Walsh Set the default_sigusr1 function to be the SIGUSR1 handler. 369f566fb1fSMichael Walsh """ 370f566fb1fSMichael Walsh 371f566fb1fSMichael Walsh gp.printn() 372f566fb1fSMichael Walsh gp.print_executing() 373f566fb1fSMichael Walsh gp.lprint_executing() 374f566fb1fSMichael Walsh signal.signal(signal.SIGUSR1, default_sigusr1) 375f566fb1fSMichael Walsh 376f566fb1fSMichael Walsh 3776741f740SMichael Walshdef setup(): 3780bbd860fSMichael Walsh r""" 3796741f740SMichael Walsh Do general program setup tasks. 3800bbd860fSMichael Walsh """ 3810bbd860fSMichael Walsh 3826741f740SMichael Walsh global cp_setup_called 38381816748SMichael Walsh global transitional_boot_selected 3840bbd860fSMichael Walsh 385b5839d00SMichael Walsh gp.qprintn() 386b5839d00SMichael Walsh 387f566fb1fSMichael Walsh set_default_siguser1() 38881816748SMichael Walsh transitional_boot_selected = False 38981816748SMichael Walsh 39083f4bc77SMichael Walsh robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 39183f4bc77SMichael Walsh repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 392d061c043SMichael Walsh # If we can't find process_plug_in_packages.py, ssh_pw or 393d061c043SMichael Walsh # validate_plug_ins.py, then we don't have our repo bin in PATH. 394004ad3c9SJoy Onyerikwu shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 395004ad3c9SJoy Onyerikwu + " ssh_pw validate_plug_ins.py", quiet=1, 396d061c043SMichael Walsh print_output=0, show_err=0) 397b5839d00SMichael Walsh if shell_rc != 0: 39883f4bc77SMichael Walsh os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 39983f4bc77SMichael Walsh # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 40083f4bc77SMichael Walsh if robot_pgm_dir_path not in sys.path: 40183f4bc77SMichael Walsh sys.path.append(robot_pgm_dir_path) 40283f4bc77SMichael Walsh PYTHONPATH = os.environ.get("PYTHONPATH", "") 40383f4bc77SMichael Walsh if PYTHONPATH == "": 40483f4bc77SMichael Walsh os.environ['PYTHONPATH'] = robot_pgm_dir_path 40583f4bc77SMichael Walsh else: 40683f4bc77SMichael Walsh os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 4076741f740SMichael Walsh 4086741f740SMichael Walsh validate_parms() 4096741f740SMichael Walsh 410c108e429SMichael Walsh gp.qprint_pgm_header() 4116741f740SMichael Walsh 412efc3ff2bSGeorge Keishing grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 41311cfc8c0SMichael Walsh 41485678948SMichael Walsh initial_plug_in_setup() 41585678948SMichael Walsh 4166741f740SMichael Walsh plug_in_setup() 4176741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 4186741f740SMichael Walsh call_point='setup') 4196741f740SMichael Walsh if rc != 0: 4206741f740SMichael Walsh error_message = "Plug-in setup failed.\n" 421c108e429SMichael Walsh gp.print_error_report(error_message) 4226741f740SMichael Walsh BuiltIn().fail(error_message) 4236741f740SMichael Walsh # Setting cp_setup_called lets our Teardown know that it needs to call 4246741f740SMichael Walsh # the cleanup plug-in call point. 4256741f740SMichael Walsh cp_setup_called = 1 4266741f740SMichael Walsh 4276741f740SMichael Walsh # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 4286741f740SMichael Walsh BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 42985678948SMichael Walsh # FFDC_LOG_PATH is used by "FFDC" keyword. 43085678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 4316741f740SMichael Walsh 432dc80d67dSMichael Walsh # Also printed by FFDC. 433dc80d67dSMichael Walsh global host_name 434dc80d67dSMichael Walsh global host_ip 435dc80d67dSMichael Walsh host = socket.gethostname() 436dc80d67dSMichael Walsh host_name, host_ip = gm.get_host_name_ip(host) 437dc80d67dSMichael Walsh 438*986d8aeeSMichael Walsh gp.dprint_var(boot_table) 439b5839d00SMichael Walsh gp.dprint_var(boot_lists) 4400bbd860fSMichael Walsh 4410bbd860fSMichael Walsh 4426741f740SMichael Walshdef validate_parms(): 4430bbd860fSMichael Walsh r""" 4446741f740SMichael Walsh Validate all program parameters. 4450bbd860fSMichael Walsh """ 4460bbd860fSMichael Walsh 447b5839d00SMichael Walsh process_pgm_parms() 4480bbd860fSMichael Walsh 449b5839d00SMichael Walsh gp.qprintn() 450b5839d00SMichael Walsh 451b5839d00SMichael Walsh global openbmc_model 4526741f740SMichael Walsh grv.rvalid_value("openbmc_host") 4536741f740SMichael Walsh grv.rvalid_value("openbmc_username") 4546741f740SMichael Walsh grv.rvalid_value("openbmc_password") 4550a3bdb4cSMichael Walsh grv.rvalid_value("rest_username") 4560a3bdb4cSMichael Walsh grv.rvalid_value("rest_password") 4570a3bdb4cSMichael Walsh grv.rvalid_value("ipmi_username") 4580a3bdb4cSMichael Walsh grv.rvalid_value("ipmi_password") 4596741f740SMichael Walsh if os_host != "": 4606741f740SMichael Walsh grv.rvalid_value("os_username") 4616741f740SMichael Walsh grv.rvalid_value("os_password") 4620bbd860fSMichael Walsh 4636741f740SMichael Walsh if pdu_host != "": 4646741f740SMichael Walsh grv.rvalid_value("pdu_username") 4656741f740SMichael Walsh grv.rvalid_value("pdu_password") 4666741f740SMichael Walsh grv.rvalid_integer("pdu_slot_no") 4676741f740SMichael Walsh if openbmc_serial_host != "": 4686741f740SMichael Walsh grv.rvalid_integer("openbmc_serial_port") 469b5839d00SMichael Walsh if openbmc_model == "": 470b5839d00SMichael Walsh status, ret_values =\ 471b5839d00SMichael Walsh grk.run_key_u("Get BMC System Model") 472b5839d00SMichael Walsh openbmc_model = ret_values 473b5839d00SMichael Walsh BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 4746741f740SMichael Walsh grv.rvalid_value("openbmc_model") 475b5839d00SMichael Walsh grv.rvalid_integer("max_num_tests") 4766741f740SMichael Walsh grv.rvalid_integer("boot_pass") 4776741f740SMichael Walsh grv.rvalid_integer("boot_fail") 4786741f740SMichael Walsh 4796741f740SMichael Walsh plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 4806741f740SMichael Walsh BuiltIn().set_global_variable("${plug_in_packages_list}", 4816741f740SMichael Walsh plug_in_packages_list) 4826741f740SMichael Walsh 483b5839d00SMichael Walsh grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 484a20da401SMichael Walsh if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 4856741f740SMichael Walsh error_message = "You must provide either a value for either the" +\ 4866741f740SMichael Walsh " boot_list or the boot_stack parm.\n" 4876741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 4886741f740SMichael Walsh 4896741f740SMichael Walsh valid_boot_list(boot_list, valid_boot_types) 4906741f740SMichael Walsh valid_boot_list(boot_stack, valid_boot_types) 4916741f740SMichael Walsh 492004ad3c9SJoy Onyerikwu selected_PDU_boots = list(set(boot_list + boot_stack) 493004ad3c9SJoy Onyerikwu & set(boot_lists['PDU_reboot'])) 49411cfc8c0SMichael Walsh 49511cfc8c0SMichael Walsh if len(selected_PDU_boots) > 0 and pdu_host == "": 49611cfc8c0SMichael Walsh error_message = "You have selected the following boots which" +\ 49711cfc8c0SMichael Walsh " require a PDU host but no value for pdu_host:\n" 49811cfc8c0SMichael Walsh error_message += gp.sprint_var(selected_PDU_boots) 499*986d8aeeSMichael Walsh error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 50011cfc8c0SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 50111cfc8c0SMichael Walsh 5026741f740SMichael Walsh return 5030bbd860fSMichael Walsh 5040bbd860fSMichael Walsh 5056741f740SMichael Walshdef my_get_state(): 5060bbd860fSMichael Walsh r""" 5076741f740SMichael Walsh Get the system state plus a little bit of wrapping. 5080bbd860fSMichael Walsh """ 5090bbd860fSMichael Walsh 5106741f740SMichael Walsh global state 5116741f740SMichael Walsh 5126741f740SMichael Walsh req_states = ['epoch_seconds'] + st.default_req_states 5136741f740SMichael Walsh 514b5839d00SMichael Walsh gp.qprint_timen("Getting system state.") 5156741f740SMichael Walsh if test_mode: 5166741f740SMichael Walsh state['epoch_seconds'] = int(time.time()) 5176741f740SMichael Walsh else: 518b5839d00SMichael Walsh state = st.get_state(req_states=req_states, quiet=quiet) 519b5839d00SMichael Walsh gp.qprint_var(state) 520341c21ebSMichael Walsh 521341c21ebSMichael Walsh 52245ca6e4cSMichael Walshdef valid_state(): 52345ca6e4cSMichael Walsh r""" 52445ca6e4cSMichael Walsh Verify that our state dictionary contains no blank values. If we don't get 52545ca6e4cSMichael Walsh valid state data, we cannot continue to work. 52645ca6e4cSMichael Walsh """ 52745ca6e4cSMichael Walsh 52845ca6e4cSMichael Walsh if st.compare_states(state, st.invalid_state_match, 'or'): 52945ca6e4cSMichael Walsh error_message = "The state dictionary contains blank fields which" +\ 53045ca6e4cSMichael Walsh " is illegal.\n" + gp.sprint_var(state) 53145ca6e4cSMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 53245ca6e4cSMichael Walsh 53345ca6e4cSMichael Walsh 5346741f740SMichael Walshdef select_boot(): 535341c21ebSMichael Walsh r""" 536341c21ebSMichael Walsh Select a boot test to be run based on our current state and return the 537341c21ebSMichael Walsh chosen boot type. 538341c21ebSMichael Walsh 539341c21ebSMichael Walsh Description of arguments: 5406741f740SMichael Walsh state The state of the machine. 541341c21ebSMichael Walsh """ 542341c21ebSMichael Walsh 54381816748SMichael Walsh global transitional_boot_selected 54430dadae2SMichael Walsh global boot_stack 54530dadae2SMichael Walsh 546b5839d00SMichael Walsh gp.qprint_timen("Selecting a boot test.") 5476741f740SMichael Walsh 54881816748SMichael Walsh if transitional_boot_selected and not boot_success: 54981816748SMichael Walsh prior_boot = next_boot 55081816748SMichael Walsh boot_candidate = boot_stack.pop() 551004ad3c9SJoy Onyerikwu gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 552004ad3c9SJoy Onyerikwu + " transition to a valid state for '" + boot_candidate 553004ad3c9SJoy Onyerikwu + "' which was at the top of the boot_stack. Since" 554004ad3c9SJoy Onyerikwu + " the '" + next_boot + "' failed, the '" 555004ad3c9SJoy Onyerikwu + boot_candidate + "' has been removed from the stack" 556004ad3c9SJoy Onyerikwu + " to avoid and endless failure loop.") 55781816748SMichael Walsh if len(boot_stack) == 0: 55881816748SMichael Walsh return "" 55981816748SMichael Walsh 5606741f740SMichael Walsh my_get_state() 56145ca6e4cSMichael Walsh valid_state() 5626741f740SMichael Walsh 56381816748SMichael Walsh transitional_boot_selected = False 5646741f740SMichael Walsh stack_popped = 0 5656741f740SMichael Walsh if len(boot_stack) > 0: 5666741f740SMichael Walsh stack_popped = 1 567b5839d00SMichael Walsh gp.qprint_dashes() 568b5839d00SMichael Walsh gp.qprint_var(boot_stack) 569b5839d00SMichael Walsh gp.qprint_dashes() 570b5839d00SMichael Walsh skip_boot_printed = 0 571b5839d00SMichael Walsh while len(boot_stack) > 0: 5726741f740SMichael Walsh boot_candidate = boot_stack.pop() 573b5839d00SMichael Walsh if stack_mode == 'normal': 574b5839d00SMichael Walsh break 575b5839d00SMichael Walsh else: 576b5839d00SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['end']): 577b5839d00SMichael Walsh if not skip_boot_printed: 578ff340006SMichael Walsh gp.qprint_var(stack_mode) 579ff340006SMichael Walsh gp.qprintn() 580004ad3c9SJoy Onyerikwu gp.qprint_timen("Skipping the following boot tests" 581004ad3c9SJoy Onyerikwu + " which are unnecessary since their" 582004ad3c9SJoy Onyerikwu + " required end states match the" 583004ad3c9SJoy Onyerikwu + " current machine state:") 584b5839d00SMichael Walsh skip_boot_printed = 1 585ff340006SMichael Walsh gp.qprint_var(boot_candidate) 586b5839d00SMichael Walsh boot_candidate = "" 587b5839d00SMichael Walsh if boot_candidate == "": 588b5839d00SMichael Walsh gp.qprint_dashes() 589b5839d00SMichael Walsh gp.qprint_var(boot_stack) 590b5839d00SMichael Walsh gp.qprint_dashes() 591b5839d00SMichael Walsh return boot_candidate 5926741f740SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['start']): 593004ad3c9SJoy Onyerikwu gp.qprint_timen("The machine state is valid for a '" 594004ad3c9SJoy Onyerikwu + boot_candidate + "' boot test.") 595b5839d00SMichael Walsh gp.qprint_dashes() 596b5839d00SMichael Walsh gp.qprint_var(boot_stack) 597b5839d00SMichael Walsh gp.qprint_dashes() 5986741f740SMichael Walsh return boot_candidate 599341c21ebSMichael Walsh else: 600004ad3c9SJoy Onyerikwu gp.qprint_timen("The machine state does not match the required" 601004ad3c9SJoy Onyerikwu + " starting state for a '" + boot_candidate 602004ad3c9SJoy Onyerikwu + "' boot test:") 603*986d8aeeSMichael Walsh gp.qprint_varx("boot_table_start_entry", 604*986d8aeeSMichael Walsh boot_table[boot_candidate]['start']) 6056741f740SMichael Walsh boot_stack.append(boot_candidate) 60681816748SMichael Walsh transitional_boot_selected = True 6076741f740SMichael Walsh popped_boot = boot_candidate 6086741f740SMichael Walsh 6096741f740SMichael Walsh # Loop through your list selecting a boot_candidates 6106741f740SMichael Walsh boot_candidates = [] 6116741f740SMichael Walsh for boot_candidate in boot_list: 6126741f740SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['start']): 6136741f740SMichael Walsh if stack_popped: 6146741f740SMichael Walsh if st.compare_states(boot_table[boot_candidate]['end'], 6156741f740SMichael Walsh boot_table[popped_boot]['start']): 6166741f740SMichael Walsh boot_candidates.append(boot_candidate) 617341c21ebSMichael Walsh else: 6186741f740SMichael Walsh boot_candidates.append(boot_candidate) 6196741f740SMichael Walsh 6206741f740SMichael Walsh if len(boot_candidates) == 0: 621004ad3c9SJoy Onyerikwu gp.qprint_timen("The user's boot list contained no boot tests" 622004ad3c9SJoy Onyerikwu + " which are valid for the current machine state.") 6236741f740SMichael Walsh boot_candidate = default_power_on 6246741f740SMichael Walsh if not st.compare_states(state, boot_table[default_power_on]['start']): 6256741f740SMichael Walsh boot_candidate = default_power_off 6266741f740SMichael Walsh boot_candidates.append(boot_candidate) 627004ad3c9SJoy Onyerikwu gp.qprint_timen("Using default '" + boot_candidate 628004ad3c9SJoy Onyerikwu + "' boot type to transition to valid state.") 6296741f740SMichael Walsh 630b5839d00SMichael Walsh gp.dprint_var(boot_candidates) 6316741f740SMichael Walsh 6326741f740SMichael Walsh # Randomly select a boot from the candidate list. 6336741f740SMichael Walsh boot = random.choice(boot_candidates) 634341c21ebSMichael Walsh 635341c21ebSMichael Walsh return boot 6360bbd860fSMichael Walsh 63755302295SMichael Walsh 638b2e53ecdSMichael Walshdef print_defect_report(ffdc_file_list): 639341c21ebSMichael Walsh r""" 640341c21ebSMichael Walsh Print a defect report. 641b2e53ecdSMichael Walsh 642b2e53ecdSMichael Walsh Description of argument(s): 643b2e53ecdSMichael Walsh ffdc_file_list A list of files which were collected by our ffdc functions. 644341c21ebSMichael Walsh """ 645341c21ebSMichael Walsh 646600876daSMichael Walsh # Making deliberate choice to NOT run plug_in_setup(). We don't want 647600876daSMichael Walsh # ffdc_prefix updated. 648600876daSMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 649600876daSMichael Walsh call_point='ffdc_report', stop_on_plug_in_failure=0) 650600876daSMichael Walsh 651e0cf8d70SMichael Walsh # Get additional header data which may have been created by ffdc plug-ins. 652e0cf8d70SMichael Walsh # Also, delete the individual header files to cleanup. 653e0cf8d70SMichael Walsh cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 654e0cf8d70SMichael Walsh " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 655e0cf8d70SMichael Walsh " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 656e0cf8d70SMichael Walsh shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 657e0cf8d70SMichael Walsh show_err=0) 658e0cf8d70SMichael Walsh 659b2e53ecdSMichael Walsh # Get additional summary data which may have been created by ffdc plug-ins. 660600876daSMichael Walsh # Also, delete the individual header files to cleanup. 661600876daSMichael Walsh cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 662600876daSMichael Walsh " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 663600876daSMichael Walsh " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 664600876daSMichael Walsh shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 665600876daSMichael Walsh show_err=0) 666600876daSMichael Walsh 667b2e53ecdSMichael Walsh # ffdc_list_file_path contains a list of any ffdc files created by plug- 668b2e53ecdSMichael Walsh # ins, etc. Read that data into a list. 669341c21ebSMichael Walsh try: 670b2e53ecdSMichael Walsh plug_in_ffdc_list = \ 671b2e53ecdSMichael Walsh open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 67236efbc04SGeorge Keishing plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 673341c21ebSMichael Walsh except IOError: 674b2e53ecdSMichael Walsh plug_in_ffdc_list = [] 675b2e53ecdSMichael Walsh 676b2e53ecdSMichael Walsh # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 677b2e53ecdSMichael Walsh # in. Eliminate duplicates and sort the list. 678004ad3c9SJoy Onyerikwu ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 679b2e53ecdSMichael Walsh 680b2e53ecdSMichael Walsh if status_file_path != "": 681b2e53ecdSMichael Walsh ffdc_file_list.insert(0, status_file_path) 682b2e53ecdSMichael Walsh 683b2e53ecdSMichael Walsh # Convert the list to a printable list. 684b2e53ecdSMichael Walsh printable_ffdc_file_list = "\n".join(ffdc_file_list) 685341c21ebSMichael Walsh 68668a61162SMichael Walsh # Open ffdc_file_list for writing. We will write a complete list of 68768a61162SMichael Walsh # FFDC files to it for possible use by plug-ins like cp_stop_check. 68868a61162SMichael Walsh ffdc_list_file = open(ffdc_list_file_path, 'w') 689b2e53ecdSMichael Walsh ffdc_list_file.write(printable_ffdc_file_list + "\n") 690b2e53ecdSMichael Walsh ffdc_list_file.close() 691b2e53ecdSMichael Walsh 692b2e53ecdSMichael Walsh indent = 0 693b2e53ecdSMichael Walsh width = 90 694b2e53ecdSMichael Walsh linefeed = 1 695b2e53ecdSMichael Walsh char = "=" 69668a61162SMichael Walsh 69768a61162SMichael Walsh gp.qprintn() 698b2e53ecdSMichael Walsh gp.qprint_dashes(indent, width, linefeed, char) 69968a61162SMichael Walsh gp.qprintn("Copy this data to the defect:\n") 70068a61162SMichael Walsh 701e0cf8d70SMichael Walsh if len(more_header_info) > 0: 702ff340006SMichael Walsh gp.qprintn(more_header_info) 703dc80d67dSMichael Walsh gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 704dc80d67dSMichael Walsh openbmc_host_name, openbmc_ip, openbmc_username, 7050a3bdb4cSMichael Walsh openbmc_password, rest_username, rest_password, ipmi_username, 7060a3bdb4cSMichael Walsh ipmi_password, os_host, os_host_name, os_ip, os_username, 707dc80d67dSMichael Walsh os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 708dc80d67dSMichael Walsh pdu_password, pdu_slot_no, openbmc_serial_host, 709dc80d67dSMichael Walsh openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 71068a61162SMichael Walsh 71168a61162SMichael Walsh gp.qprintn() 712*986d8aeeSMichael Walsh print_boot_history(boot_history) 71368a61162SMichael Walsh gp.qprintn() 71468a61162SMichael Walsh gp.qprint_var(state) 715b5839d00SMichael Walsh gp.qprintn() 716b5839d00SMichael Walsh gp.qprintn("FFDC data files:") 717b2e53ecdSMichael Walsh gp.qprintn(printable_ffdc_file_list) 718b5839d00SMichael Walsh gp.qprintn() 719341c21ebSMichael Walsh 720600876daSMichael Walsh if len(ffdc_summary_info) > 0: 721ff340006SMichael Walsh gp.qprintn(ffdc_summary_info) 722600876daSMichael Walsh 723b2e53ecdSMichael Walsh gp.qprint_dashes(indent, width, linefeed, char) 72468a61162SMichael Walsh 7256741f740SMichael Walsh 7266741f740SMichael Walshdef my_ffdc(): 7276741f740SMichael Walsh r""" 7286741f740SMichael Walsh Collect FFDC data. 7296741f740SMichael Walsh """ 7306741f740SMichael Walsh 7316741f740SMichael Walsh global state 7326741f740SMichael Walsh 7336741f740SMichael Walsh plug_in_setup() 7346741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 735600876daSMichael Walsh call_point='ffdc', stop_on_plug_in_failure=0) 7366741f740SMichael Walsh 7376741f740SMichael Walsh AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 738004ad3c9SJoy Onyerikwu status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 739004ad3c9SJoy Onyerikwu + AUTOBOOT_FFDC_PREFIX 740004ad3c9SJoy Onyerikwu + " ffdc_function_list=" 741004ad3c9SJoy Onyerikwu + ffdc_function_list, ignore=1) 74283f4bc77SMichael Walsh if status != 'PASS': 743ff340006SMichael Walsh gp.qprint_error("Call to ffdc failed.\n") 744c9bd2e87SMichael Walsh if type(ffdc_file_list) is not list: 745c9bd2e87SMichael Walsh ffdc_file_list = [] 746c9bd2e87SMichael Walsh # Leave a record for caller that "soft" errors occurred. 747c9bd2e87SMichael Walsh soft_errors = 1 748c9bd2e87SMichael Walsh gpu.save_plug_in_value(soft_errors, pgm_name) 7496741f740SMichael Walsh 7506741f740SMichael Walsh my_get_state() 7516741f740SMichael Walsh 752b2e53ecdSMichael Walsh print_defect_report(ffdc_file_list) 7536741f740SMichael Walsh 7546741f740SMichael Walsh 7556741f740SMichael Walshdef print_test_start_message(boot_keyword): 7566741f740SMichael Walsh r""" 7576741f740SMichael Walsh Print a message indicating what boot test is about to run. 7586741f740SMichael Walsh 7596741f740SMichael Walsh Description of arguments: 7606741f740SMichael Walsh boot_keyword The name of the boot which is to be run 7616741f740SMichael Walsh (e.g. "BMC Power On"). 7626741f740SMichael Walsh """ 7636741f740SMichael Walsh 764*986d8aeeSMichael Walsh global boot_history 765325eb548SSunil M global boot_start_time 7666741f740SMichael Walsh 7676741f740SMichael Walsh doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 768325eb548SSunil M 769325eb548SSunil M # Set boot_start_time for use by plug-ins. 770325eb548SSunil M boot_start_time = doing_msg[1:33] 771325eb548SSunil M gp.qprint_var(boot_start_time) 772325eb548SSunil M 773b5839d00SMichael Walsh gp.qprint(doing_msg) 7746741f740SMichael Walsh 775*986d8aeeSMichael Walsh update_boot_history(boot_history, doing_msg, max_boot_history) 7766741f740SMichael Walsh 7776741f740SMichael Walsh 778f566fb1fSMichael Walshdef stop_boot_test(signal_number=0, 779f566fb1fSMichael Walsh frame=None): 780f566fb1fSMichael Walsh r""" 781f566fb1fSMichael Walsh Handle SIGUSR1 by aborting the boot test that is running. 782f566fb1fSMichael Walsh 783f566fb1fSMichael Walsh Description of argument(s): 784f566fb1fSMichael Walsh signal_number The signal number (should always be 10 for SIGUSR1). 785f566fb1fSMichael Walsh frame The frame data. 786f566fb1fSMichael Walsh """ 787f566fb1fSMichael Walsh 788f566fb1fSMichael Walsh gp.printn() 789f566fb1fSMichael Walsh gp.print_executing() 790f566fb1fSMichael Walsh gp.lprint_executing() 791f566fb1fSMichael Walsh 792f566fb1fSMichael Walsh # Restore original sigusr1 handler. 793f566fb1fSMichael Walsh set_default_siguser1() 794f566fb1fSMichael Walsh 795f566fb1fSMichael Walsh message = "The caller has asked that the boot test be stopped and marked" 796f566fb1fSMichael Walsh message += " as a failure." 797f566fb1fSMichael Walsh 798f566fb1fSMichael Walsh function_stack = gm.get_function_stack() 799f566fb1fSMichael Walsh if "wait_state" in function_stack: 800c44aa538SMichael Walsh st.set_exit_wait_early_message(message) 801f566fb1fSMichael Walsh else: 802f566fb1fSMichael Walsh BuiltIn().fail(gp.sprint_error(message)) 803f566fb1fSMichael Walsh 804f566fb1fSMichael Walsh 8056741f740SMichael Walshdef run_boot(boot): 8066741f740SMichael Walsh r""" 8076741f740SMichael Walsh Run the specified boot. 8086741f740SMichael Walsh 8096741f740SMichael Walsh Description of arguments: 8106741f740SMichael Walsh boot The name of the boot test to be performed. 8116741f740SMichael Walsh """ 8126741f740SMichael Walsh 8136741f740SMichael Walsh global state 8146741f740SMichael Walsh 815f566fb1fSMichael Walsh signal.signal(signal.SIGUSR1, stop_boot_test) 816f566fb1fSMichael Walsh gp.qprint_timen("stop_boot_test is armed.") 817f566fb1fSMichael Walsh 8186741f740SMichael Walsh print_test_start_message(boot) 8196741f740SMichael Walsh 8206741f740SMichael Walsh plug_in_setup() 8216741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 8226741f740SMichael Walsh grpi.rprocess_plug_in_packages(call_point="pre_boot") 8236741f740SMichael Walsh if rc != 0: 8246741f740SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" +\ 825*986d8aeeSMichael Walsh gp.sprint_var(rc, fmt=gp.hexa()) 826f566fb1fSMichael Walsh set_default_siguser1() 8276741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 8286741f740SMichael Walsh 8296741f740SMichael Walsh if test_mode: 8306741f740SMichael Walsh # In test mode, we'll pretend the boot worked by assigning its 8316741f740SMichael Walsh # required end state to the default state value. 83230dadae2SMichael Walsh state = st.strip_anchor_state(boot_table[boot]['end']) 8336741f740SMichael Walsh else: 8346741f740SMichael Walsh # Assertion: We trust that the state data was made fresh by the 8356741f740SMichael Walsh # caller. 8366741f740SMichael Walsh 837b5839d00SMichael Walsh gp.qprintn() 8386741f740SMichael Walsh 8396741f740SMichael Walsh if boot_table[boot]['method_type'] == "keyword": 8400b93fbf8SMichael Walsh rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 841b5839d00SMichael Walsh boot_table[boot]['method'], 842b5839d00SMichael Walsh quiet=quiet) 8436741f740SMichael Walsh 8446741f740SMichael Walsh if boot_table[boot]['bmc_reboot']: 8456741f740SMichael Walsh st.wait_for_comm_cycle(int(state['epoch_seconds'])) 84630dadae2SMichael Walsh plug_in_setup() 84730dadae2SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 84830dadae2SMichael Walsh grpi.rprocess_plug_in_packages(call_point="post_reboot") 84930dadae2SMichael Walsh if rc != 0: 8500b93fbf8SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" 851*986d8aeeSMichael Walsh error_message += gp.sprint_var(rc, fmt=gp.hexa()) 852f566fb1fSMichael Walsh set_default_siguser1() 85330dadae2SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 8546741f740SMichael Walsh else: 8556741f740SMichael Walsh match_state = st.anchor_state(state) 8566741f740SMichael Walsh del match_state['epoch_seconds'] 8576741f740SMichael Walsh # Wait for the state to change in any way. 8586741f740SMichael Walsh st.wait_state(match_state, wait_time=state_change_timeout, 859600876daSMichael Walsh interval="10 seconds", invert=1) 8606741f740SMichael Walsh 861b5839d00SMichael Walsh gp.qprintn() 8626741f740SMichael Walsh if boot_table[boot]['end']['chassis'] == "Off": 8636741f740SMichael Walsh boot_timeout = power_off_timeout 8646741f740SMichael Walsh else: 8656741f740SMichael Walsh boot_timeout = power_on_timeout 8666741f740SMichael Walsh st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 867600876daSMichael Walsh interval="10 seconds") 8686741f740SMichael Walsh 8696741f740SMichael Walsh plug_in_setup() 8706741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 8716741f740SMichael Walsh grpi.rprocess_plug_in_packages(call_point="post_boot") 8726741f740SMichael Walsh if rc != 0: 8736741f740SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" +\ 874*986d8aeeSMichael Walsh gp.sprint_var(rc, fmt=gp.hexa()) 875f566fb1fSMichael Walsh set_default_siguser1() 8766741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 8776741f740SMichael Walsh 878f566fb1fSMichael Walsh # Restore original sigusr1 handler. 879f566fb1fSMichael Walsh set_default_siguser1() 880f566fb1fSMichael Walsh 8816741f740SMichael Walsh 8826741f740SMichael Walshdef test_loop_body(): 8836741f740SMichael Walsh r""" 8846741f740SMichael Walsh The main loop body for the loop in main_py. 8856741f740SMichael Walsh 8866741f740SMichael Walsh Description of arguments: 8876741f740SMichael Walsh boot_count The iteration number (starts at 1). 8886741f740SMichael Walsh """ 8896741f740SMichael Walsh 8906741f740SMichael Walsh global boot_count 8916741f740SMichael Walsh global state 8926741f740SMichael Walsh global next_boot 8936741f740SMichael Walsh global boot_success 894325eb548SSunil M global boot_end_time 8956741f740SMichael Walsh 896b5839d00SMichael Walsh gp.qprintn() 8976741f740SMichael Walsh 8986741f740SMichael Walsh next_boot = select_boot() 899b5839d00SMichael Walsh if next_boot == "": 900b5839d00SMichael Walsh return True 9016741f740SMichael Walsh 902b5839d00SMichael Walsh boot_count += 1 903b5839d00SMichael Walsh gp.qprint_timen("Starting boot " + str(boot_count) + ".") 9046741f740SMichael Walsh 905e0cf8d70SMichael Walsh pre_boot_plug_in_setup() 9066741f740SMichael Walsh 9076741f740SMichael Walsh cmd_buf = ["run_boot", next_boot] 9086741f740SMichael Walsh boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 9096741f740SMichael Walsh if boot_status == "FAIL": 910b5839d00SMichael Walsh gp.qprint(msg) 9116741f740SMichael Walsh 912b5839d00SMichael Walsh gp.qprintn() 9136741f740SMichael Walsh if boot_status == "PASS": 9146741f740SMichael Walsh boot_success = 1 915004ad3c9SJoy Onyerikwu completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 916004ad3c9SJoy Onyerikwu + "\" succeeded.") 9176741f740SMichael Walsh else: 9186741f740SMichael Walsh boot_success = 0 919004ad3c9SJoy Onyerikwu completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 920004ad3c9SJoy Onyerikwu + "\" failed.") 921325eb548SSunil M 922325eb548SSunil M # Set boot_end_time for use by plug-ins. 923325eb548SSunil M boot_end_time = completion_msg[1:33] 924325eb548SSunil M gp.qprint_var(boot_end_time) 925325eb548SSunil M 926325eb548SSunil M gp.qprint(completion_msg) 9276741f740SMichael Walsh 9286741f740SMichael Walsh boot_results.update(next_boot, boot_status) 9296741f740SMichael Walsh 9306741f740SMichael Walsh plug_in_setup() 9316741f740SMichael Walsh # NOTE: A post_test_case call point failure is NOT counted as a boot 9326741f740SMichael Walsh # failure. 9336741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 934600876daSMichael Walsh call_point='post_test_case', stop_on_plug_in_failure=0) 9356741f740SMichael Walsh 9366741f740SMichael Walsh plug_in_setup() 9376741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 93889de14a4SMichael Walsh call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 9396741f740SMichael Walsh stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 94012059e26SMichael Walsh if ffdc_check == "All" or\ 94189de14a4SMichael Walsh shell_rc == dump_ffdc_rc(): 94283f4bc77SMichael Walsh status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 94383f4bc77SMichael Walsh if status != 'PASS': 944ff340006SMichael Walsh gp.qprint_error("Call to my_ffdc failed.\n") 945c9bd2e87SMichael Walsh # Leave a record for caller that "soft" errors occurred. 946c9bd2e87SMichael Walsh soft_errors = 1 947c9bd2e87SMichael Walsh gpu.save_plug_in_value(soft_errors, pgm_name) 9486741f740SMichael Walsh 949aabef1e3SMichael Walsh if delete_errlogs: 950d139f286SMichael Walsh # We need to purge error logs between boots or they build up. 951b5839d00SMichael Walsh grk.run_key("Delete Error logs", ignore=1) 952d139f286SMichael Walsh 953952f9b09SMichael Walsh boot_results.print_report() 954b5839d00SMichael Walsh gp.qprint_timen("Finished boot " + str(boot_count) + ".") 955952f9b09SMichael Walsh 9566741f740SMichael Walsh plug_in_setup() 9576741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 95889de14a4SMichael Walsh call_point='stop_check', shell_rc=stop_test_rc(), 95989de14a4SMichael Walsh stop_on_non_zero_rc=1) 96089de14a4SMichael Walsh if shell_rc == stop_test_rc(): 9613ba8ecdcSMichael Walsh message = "Stopping as requested by user.\n" 9623ba8ecdcSMichael Walsh gp.print_time(message) 9633ba8ecdcSMichael Walsh BuiltIn().fail(message) 9646741f740SMichael Walsh 965d139f286SMichael Walsh # This should help prevent ConnectionErrors. 9660960b384SMichael Walsh grk.run_key_u("Close All Connections") 967d139f286SMichael Walsh 9686741f740SMichael Walsh return True 9696741f740SMichael Walsh 9706741f740SMichael Walsh 97183f4bc77SMichael Walshdef obmc_boot_test_teardown(): 9726741f740SMichael Walsh r""" 973c9116811SMichael Walsh Clean up after the Main keyword. 9746741f740SMichael Walsh """ 9756741f740SMichael Walsh 9766741f740SMichael Walsh if cp_setup_called: 9776741f740SMichael Walsh plug_in_setup() 9786741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 979600876daSMichael Walsh call_point='cleanup', stop_on_plug_in_failure=0) 9806741f740SMichael Walsh 981600876daSMichael Walsh if 'boot_results_file_path' in globals(): 982*986d8aeeSMichael Walsh # Save boot_results and boot_history objects to a file in case they are 9836c64574bSMichael Walsh # needed again. 984b5839d00SMichael Walsh gp.qprint_timen("Saving boot_results to the following path.") 985b5839d00SMichael Walsh gp.qprint_var(boot_results_file_path) 986*986d8aeeSMichael Walsh pickle.dump((boot_results, boot_history), 9876c64574bSMichael Walsh open(boot_results_file_path, 'wb'), 9880b93fbf8SMichael Walsh pickle.HIGHEST_PROTOCOL) 9890b93fbf8SMichael Walsh 990ff340006SMichael Walsh global save_stack 991ff340006SMichael Walsh # Restore any global values saved on the save_stack. 992ff340006SMichael Walsh for parm_name in main_func_parm_list: 993ff340006SMichael Walsh # Get the parm_value if it was saved on the stack. 994ff340006SMichael Walsh try: 995ff340006SMichael Walsh parm_value = save_stack.pop(parm_name) 996004ad3c9SJoy Onyerikwu except BaseException: 997ff340006SMichael Walsh # If it was not saved, no further action is required. 998ff340006SMichael Walsh continue 999ff340006SMichael Walsh 1000ff340006SMichael Walsh # Restore the saved value. 1001ff340006SMichael Walsh cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1002ff340006SMichael Walsh "}\", parm_value)" 1003ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1004ff340006SMichael Walsh exec(cmd_buf) 1005ff340006SMichael Walsh 1006ff340006SMichael Walsh gp.dprintn(save_stack.sprint_obj()) 1007ff340006SMichael Walsh 10086741f740SMichael Walsh 1009c9116811SMichael Walshdef test_teardown(): 1010c9116811SMichael Walsh r""" 1011c9116811SMichael Walsh Clean up after this test case. 1012c9116811SMichael Walsh """ 1013c9116811SMichael Walsh 1014c9116811SMichael Walsh gp.qprintn() 1015c9116811SMichael Walsh cmd_buf = ["Print Error", 1016c9116811SMichael Walsh "A keyword timeout occurred ending this program.\n"] 1017c9116811SMichael Walsh BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1018c9116811SMichael Walsh 1019c108e429SMichael Walsh gp.qprint_pgm_footer() 1020b5839d00SMichael Walsh 1021c9116811SMichael Walsh 102289de14a4SMichael Walshdef post_stack(): 102389de14a4SMichael Walsh r""" 102489de14a4SMichael Walsh Process post_stack plug-in programs. 102589de14a4SMichael Walsh """ 102689de14a4SMichael Walsh 102789de14a4SMichael Walsh if not call_post_stack_plug: 102889de14a4SMichael Walsh # The caller does not wish to have post_stack plug-in processing done. 102989de14a4SMichael Walsh return 103089de14a4SMichael Walsh 103189de14a4SMichael Walsh global boot_success 103289de14a4SMichael Walsh 103389de14a4SMichael Walsh # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 103489de14a4SMichael Walsh pre_boot_plug_in_setup() 103589de14a4SMichael Walsh # For the purposes of the following plug-ins, mark the "boot" as a success. 103689de14a4SMichael Walsh boot_success = 1 103789de14a4SMichael Walsh plug_in_setup() 1038815b1d5bSMichael Walsh rc, shell_rc, failed_plug_in_name, history =\ 1039815b1d5bSMichael Walsh grpi.rprocess_plug_in_packages(call_point='post_stack', 1040815b1d5bSMichael Walsh stop_on_plug_in_failure=0, 1041815b1d5bSMichael Walsh return_history=True) 1042*986d8aeeSMichael Walsh for doing_msg in history: 1043*986d8aeeSMichael Walsh update_boot_history(boot_history, doing_msg, max_boot_history) 1044815b1d5bSMichael Walsh if rc != 0: 1045815b1d5bSMichael Walsh boot_success = 0 104689de14a4SMichael Walsh 104789de14a4SMichael Walsh plug_in_setup() 1048815b1d5bSMichael Walsh rc, shell_rc, failed_plug_in_name =\ 1049815b1d5bSMichael Walsh grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1050815b1d5bSMichael Walsh shell_rc=dump_ffdc_rc(), 1051815b1d5bSMichael Walsh stop_on_plug_in_failure=1, 1052815b1d5bSMichael Walsh stop_on_non_zero_rc=1) 1053815b1d5bSMichael Walsh if shell_rc == dump_ffdc_rc(): 105489de14a4SMichael Walsh status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 105589de14a4SMichael Walsh if status != 'PASS': 105689de14a4SMichael Walsh gp.qprint_error("Call to my_ffdc failed.\n") 1057c9bd2e87SMichael Walsh # Leave a record for caller that "soft" errors occurred. 1058c9bd2e87SMichael Walsh soft_errors = 1 1059c9bd2e87SMichael Walsh gpu.save_plug_in_value(soft_errors, pgm_name) 106089de14a4SMichael Walsh 106189de14a4SMichael Walsh plug_in_setup() 106289de14a4SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 106389de14a4SMichael Walsh call_point='stop_check', shell_rc=stop_test_rc(), 106489de14a4SMichael Walsh stop_on_non_zero_rc=1) 106589de14a4SMichael Walsh if shell_rc == stop_test_rc(): 106689de14a4SMichael Walsh message = "Stopping as requested by user.\n" 106789de14a4SMichael Walsh gp.print_time(message) 106889de14a4SMichael Walsh BuiltIn().fail(message) 106989de14a4SMichael Walsh 107089de14a4SMichael Walsh 1071ff340006SMichael Walshdef obmc_boot_test_py(loc_boot_stack=None, 1072ff340006SMichael Walsh loc_stack_mode=None, 1073ff340006SMichael Walsh loc_quiet=None): 10746741f740SMichael Walsh r""" 10756741f740SMichael Walsh Do main program processing. 10766741f740SMichael Walsh """ 10776741f740SMichael Walsh 1078ff340006SMichael Walsh global save_stack 1079ff340006SMichael Walsh 108036efbc04SGeorge Keishing gp.dprintn() 1081ff340006SMichael Walsh # Process function parms. 1082ff340006SMichael Walsh for parm_name in main_func_parm_list: 1083ff340006SMichael Walsh # Get parm's value. 108436efbc04SGeorge Keishing parm_value = eval("loc_" + parm_name) 108536efbc04SGeorge Keishing gp.dpvars(parm_name, parm_value) 1086ff340006SMichael Walsh 108736efbc04SGeorge Keishing if parm_value is not None: 1088ff340006SMichael Walsh # Save the global value on a stack. 1089ff340006SMichael Walsh cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1090ff340006SMichael Walsh parm_name + "}\"), \"" + parm_name + "\")" 1091ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1092ff340006SMichael Walsh exec(cmd_buf) 1093ff340006SMichael Walsh 1094ff340006SMichael Walsh # Set the global value to the passed value. 1095ff340006SMichael Walsh cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1096ff340006SMichael Walsh "}\", loc_" + parm_name + ")" 1097ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1098ff340006SMichael Walsh exec(cmd_buf) 1099ff340006SMichael Walsh 1100ff340006SMichael Walsh gp.dprintn(save_stack.sprint_obj()) 1101b5839d00SMichael Walsh 11026741f740SMichael Walsh setup() 11036741f740SMichael Walsh 1104cd9fbfd7SMichael Walsh init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1105cd9fbfd7SMichael Walsh 1106a20da401SMichael Walsh if ffdc_only: 1107a20da401SMichael Walsh gp.qprint_timen("Caller requested ffdc_only.") 1108*986d8aeeSMichael Walsh if do_pre_boot_plug_in_setup: 1109e0cf8d70SMichael Walsh pre_boot_plug_in_setup() 111083f4bc77SMichael Walsh grk.run_key_u("my_ffdc") 1111764d2f83SMichael Walsh return 1112a20da401SMichael Walsh 11136741f740SMichael Walsh # Process caller's boot_stack. 11146741f740SMichael Walsh while (len(boot_stack) > 0): 11156741f740SMichael Walsh test_loop_body() 11166741f740SMichael Walsh 1117b5839d00SMichael Walsh gp.qprint_timen("Finished processing stack.") 111830dadae2SMichael Walsh 111989de14a4SMichael Walsh post_stack() 112089de14a4SMichael Walsh 11216741f740SMichael Walsh # Process caller's boot_list. 11226741f740SMichael Walsh if len(boot_list) > 0: 11236741f740SMichael Walsh for ix in range(1, max_num_tests + 1): 11246741f740SMichael Walsh test_loop_body() 11256741f740SMichael Walsh 1126b5839d00SMichael Walsh gp.qprint_timen("Completed all requested boot tests.") 1127b5839d00SMichael Walsh 1128b5839d00SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 1129cd9fbfd7SMichael Walsh new_fail = boot_fail - init_boot_fail 1130cd9fbfd7SMichael Walsh if new_fail > boot_fail_threshold: 1131b5839d00SMichael Walsh error_message = "Boot failures exceed the boot failure" +\ 1132b5839d00SMichael Walsh " threshold:\n" +\ 1133cd9fbfd7SMichael Walsh gp.sprint_var(new_fail) +\ 1134b5839d00SMichael Walsh gp.sprint_var(boot_fail_threshold) 1135b5839d00SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 1136