10bbd860fSMichael Walsh#!/usr/bin/env python 20bbd860fSMichael Walsh 30bbd860fSMichael Walshr""" 40bbd860fSMichael WalshThis module is the python counterpart to obmc_boot_test. 50bbd860fSMichael Walsh""" 60bbd860fSMichael Walsh 70b93fbf8SMichael Walshimport os 80b93fbf8SMichael Walshimport imp 90b93fbf8SMichael Walshimport time 100b93fbf8SMichael Walshimport glob 110b93fbf8SMichael Walshimport random 120ad0f7f8SMichael Walshimport re 13f566fb1fSMichael Walshimport signal 14d54bbc22SGeorge Keishingtry: 150b93fbf8SMichael Walsh import cPickle as pickle 16d54bbc22SGeorge Keishingexcept ImportError: 17d54bbc22SGeorge Keishing import pickle 18dc80d67dSMichael Walshimport socket 190b93fbf8SMichael Walsh 200b93fbf8SMichael Walshfrom robot.utils import DotDict 210b93fbf8SMichael Walshfrom robot.libraries.BuiltIn import BuiltIn 220b93fbf8SMichael Walsh 236741f740SMichael Walshfrom boot_data import * 24c9116811SMichael Walshimport gen_print as gp 250bbd860fSMichael Walshimport gen_robot_print as grp 2655302295SMichael Walshimport gen_robot_plug_in as grpi 276741f740SMichael Walshimport gen_robot_valid as grv 286741f740SMichael Walshimport gen_misc as gm 296741f740SMichael Walshimport gen_cmd as gc 30b5839d00SMichael Walshimport gen_robot_keyword as grk 3155302295SMichael Walshimport state as st 32ff340006SMichael Walshimport var_stack as vs 330bbd860fSMichael Walsh 340b93fbf8SMichael Walshbase_path = os.path.dirname(os.path.dirname( 350b93fbf8SMichael Walsh imp.find_module("gen_robot_print")[1])) +\ 360b93fbf8SMichael Walsh os.sep 370b93fbf8SMichael Walshsys.path.append(base_path + "extended/") 380b93fbf8SMichael Walshimport run_keyword as rk 390bbd860fSMichael Walsh 40e1e26448SMichael Walsh# Setting master_pid correctly influences the behavior of plug-ins like 41e1e26448SMichael Walsh# DB_Logging 42e1e26448SMichael Walshprogram_pid = os.getpid() 43e1e26448SMichael Walshmaster_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 44004ad3c9SJoy Onyerikwupgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 45e1e26448SMichael Walsh 46b5839d00SMichael Walsh# Set up boot data structures. 47b5839d00SMichael Walshboot_table = create_boot_table() 48b5839d00SMichael Walshvalid_boot_types = create_valid_boot_list(boot_table) 490b93fbf8SMichael Walsh 506741f740SMichael Walshboot_lists = read_boot_lists() 51815b1d5bSMichael Walsh# The maximum number of entries that can be in the last_ten global variable. 52815b1d5bSMichael Walshmax_boot_history = 10 536741f740SMichael Walshlast_ten = [] 546741f740SMichael Walsh 557dc885b6SMichael Walshstate = st.return_state_constant('default_state') 566741f740SMichael Walshcp_setup_called = 0 576741f740SMichael Walshnext_boot = "" 586741f740SMichael Walshbase_tool_dir_path = os.path.normpath(os.environ.get( 596741f740SMichael Walsh 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 60b5839d00SMichael Walsh 616741f740SMichael Walshffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 626741f740SMichael Walshboot_success = 0 636741f740SMichael Walshstatus_dir_path = os.environ.get('STATUS_DIR_PATH', "") 646741f740SMichael Walshif status_dir_path != "": 656741f740SMichael Walsh status_dir_path = os.path.normpath(status_dir_path) + os.sep 660b93fbf8SMichael Walshdefault_power_on = "REST Power On" 670b93fbf8SMichael Walshdefault_power_off = "REST Power Off" 686741f740SMichael Walshboot_count = 0 690bbd860fSMichael Walsh 7085678948SMichael WalshLOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 71e1974b96SMichael Walshffdc_prefix = "" 72325eb548SSunil Mboot_start_time = "" 73325eb548SSunil Mboot_end_time = "" 74ff340006SMichael Walshsave_stack = vs.var_stack('save_stack') 75ff340006SMichael Walshmain_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 7685678948SMichael Walsh 7785678948SMichael Walsh 7889de14a4SMichael Walshdef dump_ffdc_rc(): 7989de14a4SMichael Walsh r""" 8089de14a4SMichael Walsh Return the constant dump ffdc test return code value. 8189de14a4SMichael Walsh 8289de14a4SMichael Walsh When a plug-in call point program returns this value, it indicates that 8389de14a4SMichael Walsh this program should collect FFDC. 8489de14a4SMichael Walsh """ 8589de14a4SMichael Walsh 8689de14a4SMichael Walsh return 0x00000200 8789de14a4SMichael Walsh 8889de14a4SMichael Walsh 8989de14a4SMichael Walshdef stop_test_rc(): 9089de14a4SMichael Walsh r""" 9189de14a4SMichael Walsh Return the constant stop test return code value. 9289de14a4SMichael Walsh 9389de14a4SMichael Walsh When a plug-in call point program returns this value, it indicates that 9489de14a4SMichael Walsh this program should stop running. 9589de14a4SMichael Walsh """ 9689de14a4SMichael Walsh 9789de14a4SMichael Walsh return 0x00000200 9889de14a4SMichael Walsh 9989de14a4SMichael Walsh 1000ad0f7f8SMichael Walshdef process_host(host, 1010ad0f7f8SMichael Walsh host_var_name=""): 1020ad0f7f8SMichael Walsh r""" 1030ad0f7f8SMichael Walsh Process a host by getting the associated host name and IP address and 1040ad0f7f8SMichael Walsh setting them in global variables. 1050ad0f7f8SMichael Walsh 1060ad0f7f8SMichael Walsh If the caller does not pass the host_var_name, this function will try to 1070ad0f7f8SMichael Walsh figure out the name of the variable used by the caller for the host parm. 1080ad0f7f8SMichael Walsh Callers are advised to explicitly specify the host_var_name when calling 1090ad0f7f8SMichael Walsh with an exec command. In such cases, the get_arg_name cannot figure out 1100ad0f7f8SMichael Walsh the host variable name. 1110ad0f7f8SMichael Walsh 1120ad0f7f8SMichael Walsh This function will then create similar global variable names by 1130ad0f7f8SMichael Walsh removing "_host" and appending "_host_name" or "_ip" to the host variable 1140ad0f7f8SMichael Walsh name. 1150ad0f7f8SMichael Walsh 1160ad0f7f8SMichael Walsh Example: 1170ad0f7f8SMichael Walsh 1180ad0f7f8SMichael Walsh If a call is made like this: 1190ad0f7f8SMichael Walsh process_host(openbmc_host) 1200ad0f7f8SMichael Walsh 1210ad0f7f8SMichael Walsh Global variables openbmc_host_name and openbmc_ip will be set. 1220ad0f7f8SMichael Walsh 1230ad0f7f8SMichael Walsh Description of argument(s): 1240ad0f7f8SMichael Walsh host A host name or IP. The name of the variable used should 1250ad0f7f8SMichael Walsh have a suffix of "_host". 1260ad0f7f8SMichael Walsh host_var_name The name of the variable being used as the host parm. 1270ad0f7f8SMichael Walsh """ 1280ad0f7f8SMichael Walsh 1290ad0f7f8SMichael Walsh if host_var_name == "": 1300ad0f7f8SMichael Walsh host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 1310ad0f7f8SMichael Walsh 1320ad0f7f8SMichael Walsh host_name_var_name = re.sub("host", "host_name", host_var_name) 1330ad0f7f8SMichael Walsh ip_var_name = re.sub("host", "ip", host_var_name) 1340ad0f7f8SMichael Walsh cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 1350ad0f7f8SMichael Walsh host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 1360ad0f7f8SMichael Walsh host + "')" 1370ad0f7f8SMichael Walsh exec(cmd_buf) 1380ad0f7f8SMichael Walsh 1390ad0f7f8SMichael Walsh 140b5839d00SMichael Walshdef process_pgm_parms(): 141b5839d00SMichael Walsh r""" 142b5839d00SMichael Walsh Process the program parameters by assigning them all to corresponding 143b5839d00SMichael Walsh globals. Also, set some global values that depend on program parameters. 144b5839d00SMichael Walsh """ 145b5839d00SMichael Walsh 146b5839d00SMichael Walsh # Program parameter processing. 147b5839d00SMichael Walsh # Assign all program parms to python variables which are global to this 148b5839d00SMichael Walsh # module. 149b5839d00SMichael Walsh 150b5839d00SMichael Walsh global parm_list 151b5839d00SMichael Walsh parm_list = BuiltIn().get_variable_value("${parm_list}") 152b5839d00SMichael Walsh # The following subset of parms should be processed as integers. 153b5839d00SMichael Walsh int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 15489de14a4SMichael Walsh 'boot_fail_threshold', 'delete_errlogs', 15589de14a4SMichael Walsh 'call_post_stack_plug', 'quiet', 'test_mode', 'debug'] 156b5839d00SMichael Walsh for parm in parm_list: 157b5839d00SMichael Walsh if parm in int_list: 158b5839d00SMichael Walsh sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 159b5839d00SMichael Walsh "}\", \"0\"))" 160b5839d00SMichael Walsh else: 161b5839d00SMichael Walsh sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 162b5839d00SMichael Walsh cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 163ff340006SMichael Walsh gp.dpissuing(cmd_buf) 164b5839d00SMichael Walsh exec(cmd_buf) 1650ad0f7f8SMichael Walsh if re.match(r".*_host$", parm): 1660ad0f7f8SMichael Walsh cmd_buf = "process_host(" + parm + ", '" + parm + "')" 1670ad0f7f8SMichael Walsh exec(cmd_buf) 1680ad0f7f8SMichael Walsh if re.match(r".*_password$", parm): 1690ad0f7f8SMichael Walsh # Register the value of any parm whose name ends in _password. 1700ad0f7f8SMichael Walsh # This will cause the print functions to replace passwords with 1710ad0f7f8SMichael Walsh # asterisks in the output. 1720ad0f7f8SMichael Walsh cmd_buf = "gp.register_passwords(" + parm + ")" 1730ad0f7f8SMichael Walsh exec(cmd_buf) 174b5839d00SMichael Walsh 175b5839d00SMichael Walsh global ffdc_dir_path_style 176b5839d00SMichael Walsh global boot_list 177b5839d00SMichael Walsh global boot_stack 178b5839d00SMichael Walsh global boot_results_file_path 179b5839d00SMichael Walsh global boot_results 1806c64574bSMichael Walsh global last_ten 181b5839d00SMichael Walsh global ffdc_list_file_path 182e0cf8d70SMichael Walsh global ffdc_report_list_path 183600876daSMichael Walsh global ffdc_summary_list_path 184b5839d00SMichael Walsh 185b5839d00SMichael Walsh if ffdc_dir_path_style == "": 186b5839d00SMichael Walsh ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 187b5839d00SMichael Walsh 188b5839d00SMichael Walsh # Convert these program parms to lists for easier processing.. 18936efbc04SGeorge Keishing boot_list = list(filter(None, boot_list.split(":"))) 19036efbc04SGeorge Keishing boot_stack = list(filter(None, boot_stack.split(":"))) 191b5839d00SMichael Walsh 192903e0b20SMichael Walsh cleanup_boot_results_file() 193903e0b20SMichael Walsh boot_results_file_path = create_boot_results_file_path(pgm_name, 194903e0b20SMichael Walsh openbmc_nickname, 195903e0b20SMichael Walsh master_pid) 196b5839d00SMichael Walsh 197b5839d00SMichael Walsh if os.path.isfile(boot_results_file_path): 198b5839d00SMichael Walsh # We've been called before in this run so we'll load the saved 1996c64574bSMichael Walsh # boot_results and last_ten objects. 2006c64574bSMichael Walsh boot_results, last_ten =\ 2016c64574bSMichael Walsh pickle.load(open(boot_results_file_path, 'rb')) 202b5839d00SMichael Walsh else: 203b5839d00SMichael Walsh boot_results = boot_results(boot_table, boot_pass, boot_fail) 204b5839d00SMichael Walsh 205b5839d00SMichael Walsh ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 206b5839d00SMichael Walsh "/FFDC_FILE_LIST" 207e0cf8d70SMichael Walsh ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 208e0cf8d70SMichael Walsh "/FFDC_REPORT_FILE_LIST" 209b5839d00SMichael Walsh 210600876daSMichael Walsh ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 211600876daSMichael Walsh "/FFDC_SUMMARY_FILE_LIST" 212600876daSMichael Walsh 213b5839d00SMichael Walsh 21485678948SMichael Walshdef initial_plug_in_setup(): 21585678948SMichael Walsh r""" 21685678948SMichael Walsh Initialize all plug-in environment variables which do not change for the 21785678948SMichael Walsh duration of the program. 21885678948SMichael Walsh 21985678948SMichael Walsh """ 22085678948SMichael Walsh 22185678948SMichael Walsh global LOG_LEVEL 22285678948SMichael Walsh BuiltIn().set_log_level("NONE") 22385678948SMichael Walsh 22485678948SMichael Walsh BuiltIn().set_global_variable("${master_pid}", master_pid) 22585678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 22685678948SMichael Walsh BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 22785678948SMichael Walsh BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 22885678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 22985678948SMichael Walsh ffdc_list_file_path) 230e0cf8d70SMichael Walsh BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 231e0cf8d70SMichael Walsh ffdc_report_list_path) 232600876daSMichael Walsh BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 233600876daSMichael Walsh ffdc_summary_list_path) 23485678948SMichael Walsh 23585678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 23685678948SMichael Walsh ffdc_dir_path_style) 23785678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_CHECK}", 23885678948SMichael Walsh ffdc_check) 23985678948SMichael Walsh 24085678948SMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 24185678948SMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 24285678948SMichael Walsh # element in additional_values. 24385678948SMichael Walsh additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 24485678948SMichael Walsh "status_dir_path", "base_tool_dir_path", 245600876daSMichael Walsh "ffdc_list_file_path", "ffdc_report_list_path", 2460a3bdb4cSMichael Walsh "ffdc_summary_list_path", "execdir"] 24785678948SMichael Walsh 24885678948SMichael Walsh plug_in_vars = parm_list + additional_values 24985678948SMichael Walsh 25085678948SMichael Walsh for var_name in plug_in_vars: 25185678948SMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 25285678948SMichael Walsh var_name = var_name.upper() 25385678948SMichael Walsh if var_value is None: 25485678948SMichael Walsh var_value = "" 25585678948SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 25685678948SMichael Walsh 25785678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 25885678948SMichael Walsh 25968a61162SMichael Walsh # Make sure the ffdc list directory exists. 26068a61162SMichael Walsh ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 26168a61162SMichael Walsh if not os.path.exists(ffdc_list_dir_path): 26268a61162SMichael Walsh os.makedirs(ffdc_list_dir_path) 26385678948SMichael Walsh 26485678948SMichael Walsh 2650bbd860fSMichael Walshdef plug_in_setup(): 2660bbd860fSMichael Walsh r""" 26785678948SMichael Walsh Initialize all changing plug-in environment variables for use by the 26885678948SMichael Walsh plug-in programs. 2690bbd860fSMichael Walsh """ 2700bbd860fSMichael Walsh 27185678948SMichael Walsh global LOG_LEVEL 27285678948SMichael Walsh global test_really_running 27385678948SMichael Walsh 27485678948SMichael Walsh BuiltIn().set_log_level("NONE") 27585678948SMichael Walsh 2766741f740SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 2770bbd860fSMichael Walsh if boot_pass > 1: 2780bbd860fSMichael Walsh test_really_running = 1 2790bbd860fSMichael Walsh else: 2800bbd860fSMichael Walsh test_really_running = 0 2810bbd860fSMichael Walsh 2826741f740SMichael Walsh BuiltIn().set_global_variable("${test_really_running}", 2836741f740SMichael Walsh test_really_running) 2846741f740SMichael Walsh BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 2856741f740SMichael Walsh BuiltIn().set_global_variable("${boot_pass}", boot_pass) 2866741f740SMichael Walsh BuiltIn().set_global_variable("${boot_fail}", boot_fail) 2876741f740SMichael Walsh BuiltIn().set_global_variable("${boot_success}", boot_success) 2886741f740SMichael Walsh BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 289325eb548SSunil M BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 290325eb548SSunil M BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 2914c9a6453SMichael Walsh 2920bbd860fSMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 2930bbd860fSMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 2940bbd860fSMichael Walsh # element in additional_values. 2950bbd860fSMichael Walsh additional_values = ["boot_type_desc", "boot_success", "boot_pass", 296325eb548SSunil M "boot_fail", "test_really_running", "ffdc_prefix", 297325eb548SSunil M "boot_start_time", "boot_end_time"] 2980bbd860fSMichael Walsh 29985678948SMichael Walsh plug_in_vars = additional_values 3000bbd860fSMichael Walsh 3010bbd860fSMichael Walsh for var_name in plug_in_vars: 3020bbd860fSMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 3030bbd860fSMichael Walsh var_name = var_name.upper() 3040bbd860fSMichael Walsh if var_value is None: 3050bbd860fSMichael Walsh var_value = "" 3066741f740SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 3070bbd860fSMichael Walsh 3080bbd860fSMichael Walsh if debug: 3096741f740SMichael Walsh shell_rc, out_buf = \ 3106741f740SMichael Walsh gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 3110bbd860fSMichael Walsh 31285678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 31385678948SMichael Walsh 3140bbd860fSMichael Walsh 315e0cf8d70SMichael Walshdef pre_boot_plug_in_setup(): 316e0cf8d70SMichael Walsh 317e0cf8d70SMichael Walsh # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 318e0cf8d70SMichael Walsh try: 319e0cf8d70SMichael Walsh os.remove(ffdc_list_file_path) 320e0cf8d70SMichael Walsh except OSError: 321e0cf8d70SMichael Walsh pass 322e0cf8d70SMichael Walsh 323e0cf8d70SMichael Walsh # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 324e0cf8d70SMichael Walsh try: 325e0cf8d70SMichael Walsh os.remove(ffdc_report_list_path) 326e0cf8d70SMichael Walsh except OSError: 327e0cf8d70SMichael Walsh pass 328e0cf8d70SMichael Walsh 329600876daSMichael Walsh # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 330600876daSMichael Walsh try: 331600876daSMichael Walsh os.remove(ffdc_summary_list_path) 332600876daSMichael Walsh except OSError: 333600876daSMichael Walsh pass 334600876daSMichael Walsh 335e1974b96SMichael Walsh global ffdc_prefix 336e1974b96SMichael Walsh 337e1974b96SMichael Walsh seconds = time.time() 338e1974b96SMichael Walsh loc_time = time.localtime(seconds) 339e1974b96SMichael Walsh time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 340e1974b96SMichael Walsh 341e1974b96SMichael Walsh ffdc_prefix = openbmc_nickname + "." + time_string 342e1974b96SMichael Walsh 343e0cf8d70SMichael Walsh 344f566fb1fSMichael Walshdef default_sigusr1(signal_number=0, 345f566fb1fSMichael Walsh frame=None): 346f566fb1fSMichael Walsh r""" 347f566fb1fSMichael Walsh Handle SIGUSR1 by doing nothing. 348f566fb1fSMichael Walsh 349f566fb1fSMichael Walsh This function assists in debugging SIGUSR1 processing by printing messages 350f566fb1fSMichael Walsh to stdout and to the log.html file. 351f566fb1fSMichael Walsh 352f566fb1fSMichael Walsh Description of argument(s): 353f566fb1fSMichael Walsh signal_number The signal number (should always be 10 for SIGUSR1). 354f566fb1fSMichael Walsh frame The frame data. 355f566fb1fSMichael Walsh """ 356f566fb1fSMichael Walsh 357f566fb1fSMichael Walsh gp.printn() 358f566fb1fSMichael Walsh gp.print_executing() 359f566fb1fSMichael Walsh gp.lprint_executing() 360f566fb1fSMichael Walsh 361f566fb1fSMichael Walsh 362f566fb1fSMichael Walshdef set_default_siguser1(): 363f566fb1fSMichael Walsh r""" 364f566fb1fSMichael Walsh Set the default_sigusr1 function to be the SIGUSR1 handler. 365f566fb1fSMichael Walsh """ 366f566fb1fSMichael Walsh 367f566fb1fSMichael Walsh gp.printn() 368f566fb1fSMichael Walsh gp.print_executing() 369f566fb1fSMichael Walsh gp.lprint_executing() 370f566fb1fSMichael Walsh signal.signal(signal.SIGUSR1, default_sigusr1) 371f566fb1fSMichael Walsh 372f566fb1fSMichael Walsh 3736741f740SMichael Walshdef setup(): 3740bbd860fSMichael Walsh r""" 3756741f740SMichael Walsh Do general program setup tasks. 3760bbd860fSMichael Walsh """ 3770bbd860fSMichael Walsh 3786741f740SMichael Walsh global cp_setup_called 37981816748SMichael Walsh global transitional_boot_selected 3800bbd860fSMichael Walsh 381b5839d00SMichael Walsh gp.qprintn() 382b5839d00SMichael Walsh 383f566fb1fSMichael Walsh set_default_siguser1() 38481816748SMichael Walsh transitional_boot_selected = False 38581816748SMichael Walsh 38683f4bc77SMichael Walsh robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 38783f4bc77SMichael Walsh repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 388d061c043SMichael Walsh # If we can't find process_plug_in_packages.py, ssh_pw or 389d061c043SMichael Walsh # validate_plug_ins.py, then we don't have our repo bin in PATH. 390004ad3c9SJoy Onyerikwu shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 391004ad3c9SJoy Onyerikwu + " ssh_pw validate_plug_ins.py", quiet=1, 392d061c043SMichael Walsh print_output=0, show_err=0) 393b5839d00SMichael Walsh if shell_rc != 0: 39483f4bc77SMichael Walsh os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 39583f4bc77SMichael Walsh # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 39683f4bc77SMichael Walsh if robot_pgm_dir_path not in sys.path: 39783f4bc77SMichael Walsh sys.path.append(robot_pgm_dir_path) 39883f4bc77SMichael Walsh PYTHONPATH = os.environ.get("PYTHONPATH", "") 39983f4bc77SMichael Walsh if PYTHONPATH == "": 40083f4bc77SMichael Walsh os.environ['PYTHONPATH'] = robot_pgm_dir_path 40183f4bc77SMichael Walsh else: 40283f4bc77SMichael Walsh os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 4036741f740SMichael Walsh 4046741f740SMichael Walsh validate_parms() 4056741f740SMichael Walsh 4066741f740SMichael Walsh grp.rqprint_pgm_header() 4076741f740SMichael Walsh 408efc3ff2bSGeorge Keishing grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 40911cfc8c0SMichael Walsh 41085678948SMichael Walsh initial_plug_in_setup() 41185678948SMichael Walsh 4126741f740SMichael Walsh plug_in_setup() 4136741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 4146741f740SMichael Walsh call_point='setup') 4156741f740SMichael Walsh if rc != 0: 4166741f740SMichael Walsh error_message = "Plug-in setup failed.\n" 4176741f740SMichael Walsh grp.rprint_error_report(error_message) 4186741f740SMichael Walsh BuiltIn().fail(error_message) 4196741f740SMichael Walsh # Setting cp_setup_called lets our Teardown know that it needs to call 4206741f740SMichael Walsh # the cleanup plug-in call point. 4216741f740SMichael Walsh cp_setup_called = 1 4226741f740SMichael Walsh 4236741f740SMichael Walsh # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 4246741f740SMichael Walsh BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 42585678948SMichael Walsh # FFDC_LOG_PATH is used by "FFDC" keyword. 42685678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 4276741f740SMichael Walsh 428dc80d67dSMichael Walsh # Also printed by FFDC. 429dc80d67dSMichael Walsh global host_name 430dc80d67dSMichael Walsh global host_ip 431dc80d67dSMichael Walsh host = socket.gethostname() 432dc80d67dSMichael Walsh host_name, host_ip = gm.get_host_name_ip(host) 433dc80d67dSMichael Walsh 434b5839d00SMichael Walsh gp.dprint_var(boot_table, 1) 435b5839d00SMichael Walsh gp.dprint_var(boot_lists) 4360bbd860fSMichael Walsh 4370bbd860fSMichael Walsh 4386741f740SMichael Walshdef validate_parms(): 4390bbd860fSMichael Walsh r""" 4406741f740SMichael Walsh Validate all program parameters. 4410bbd860fSMichael Walsh """ 4420bbd860fSMichael Walsh 443b5839d00SMichael Walsh process_pgm_parms() 4440bbd860fSMichael Walsh 445b5839d00SMichael Walsh gp.qprintn() 446b5839d00SMichael Walsh 447b5839d00SMichael Walsh global openbmc_model 4486741f740SMichael Walsh grv.rvalid_value("openbmc_host") 4496741f740SMichael Walsh grv.rvalid_value("openbmc_username") 4506741f740SMichael Walsh grv.rvalid_value("openbmc_password") 4510a3bdb4cSMichael Walsh grv.rvalid_value("rest_username") 4520a3bdb4cSMichael Walsh grv.rvalid_value("rest_password") 4530a3bdb4cSMichael Walsh grv.rvalid_value("ipmi_username") 4540a3bdb4cSMichael Walsh grv.rvalid_value("ipmi_password") 4556741f740SMichael Walsh if os_host != "": 4566741f740SMichael Walsh grv.rvalid_value("os_username") 4576741f740SMichael Walsh grv.rvalid_value("os_password") 4580bbd860fSMichael Walsh 4596741f740SMichael Walsh if pdu_host != "": 4606741f740SMichael Walsh grv.rvalid_value("pdu_username") 4616741f740SMichael Walsh grv.rvalid_value("pdu_password") 4626741f740SMichael Walsh grv.rvalid_integer("pdu_slot_no") 4636741f740SMichael Walsh if openbmc_serial_host != "": 4646741f740SMichael Walsh grv.rvalid_integer("openbmc_serial_port") 465b5839d00SMichael Walsh if openbmc_model == "": 466b5839d00SMichael Walsh status, ret_values =\ 467b5839d00SMichael Walsh grk.run_key_u("Get BMC System Model") 468b5839d00SMichael Walsh openbmc_model = ret_values 469b5839d00SMichael Walsh BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 4706741f740SMichael Walsh grv.rvalid_value("openbmc_model") 471b5839d00SMichael Walsh grv.rvalid_integer("max_num_tests") 4726741f740SMichael Walsh grv.rvalid_integer("boot_pass") 4736741f740SMichael Walsh grv.rvalid_integer("boot_fail") 4746741f740SMichael Walsh 4756741f740SMichael Walsh plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 4766741f740SMichael Walsh BuiltIn().set_global_variable("${plug_in_packages_list}", 4776741f740SMichael Walsh plug_in_packages_list) 4786741f740SMichael Walsh 479b5839d00SMichael Walsh grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 480a20da401SMichael Walsh if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 4816741f740SMichael Walsh error_message = "You must provide either a value for either the" +\ 4826741f740SMichael Walsh " boot_list or the boot_stack parm.\n" 4836741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 4846741f740SMichael Walsh 4856741f740SMichael Walsh valid_boot_list(boot_list, valid_boot_types) 4866741f740SMichael Walsh valid_boot_list(boot_stack, valid_boot_types) 4876741f740SMichael Walsh 488004ad3c9SJoy Onyerikwu selected_PDU_boots = list(set(boot_list + boot_stack) 489004ad3c9SJoy Onyerikwu & set(boot_lists['PDU_reboot'])) 49011cfc8c0SMichael Walsh 49111cfc8c0SMichael Walsh if len(selected_PDU_boots) > 0 and pdu_host == "": 49211cfc8c0SMichael Walsh error_message = "You have selected the following boots which" +\ 49311cfc8c0SMichael Walsh " require a PDU host but no value for pdu_host:\n" 49411cfc8c0SMichael Walsh error_message += gp.sprint_var(selected_PDU_boots) 49511cfc8c0SMichael Walsh error_message += gp.sprint_var(pdu_host, 2) 49611cfc8c0SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 49711cfc8c0SMichael Walsh 4986741f740SMichael Walsh return 4990bbd860fSMichael Walsh 5000bbd860fSMichael Walsh 5016741f740SMichael Walshdef my_get_state(): 5020bbd860fSMichael Walsh r""" 5036741f740SMichael Walsh Get the system state plus a little bit of wrapping. 5040bbd860fSMichael Walsh """ 5050bbd860fSMichael Walsh 5066741f740SMichael Walsh global state 5076741f740SMichael Walsh 5086741f740SMichael Walsh req_states = ['epoch_seconds'] + st.default_req_states 5096741f740SMichael Walsh 510b5839d00SMichael Walsh gp.qprint_timen("Getting system state.") 5116741f740SMichael Walsh if test_mode: 5126741f740SMichael Walsh state['epoch_seconds'] = int(time.time()) 5136741f740SMichael Walsh else: 514b5839d00SMichael Walsh state = st.get_state(req_states=req_states, quiet=quiet) 515b5839d00SMichael Walsh gp.qprint_var(state) 516341c21ebSMichael Walsh 517341c21ebSMichael Walsh 51845ca6e4cSMichael Walshdef valid_state(): 51945ca6e4cSMichael Walsh r""" 52045ca6e4cSMichael Walsh Verify that our state dictionary contains no blank values. If we don't get 52145ca6e4cSMichael Walsh valid state data, we cannot continue to work. 52245ca6e4cSMichael Walsh """ 52345ca6e4cSMichael Walsh 52445ca6e4cSMichael Walsh if st.compare_states(state, st.invalid_state_match, 'or'): 52545ca6e4cSMichael Walsh error_message = "The state dictionary contains blank fields which" +\ 52645ca6e4cSMichael Walsh " is illegal.\n" + gp.sprint_var(state) 52745ca6e4cSMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 52845ca6e4cSMichael Walsh 52945ca6e4cSMichael Walsh 5306741f740SMichael Walshdef select_boot(): 531341c21ebSMichael Walsh r""" 532341c21ebSMichael Walsh Select a boot test to be run based on our current state and return the 533341c21ebSMichael Walsh chosen boot type. 534341c21ebSMichael Walsh 535341c21ebSMichael Walsh Description of arguments: 5366741f740SMichael Walsh state The state of the machine. 537341c21ebSMichael Walsh """ 538341c21ebSMichael Walsh 53981816748SMichael Walsh global transitional_boot_selected 54030dadae2SMichael Walsh global boot_stack 54130dadae2SMichael Walsh 542b5839d00SMichael Walsh gp.qprint_timen("Selecting a boot test.") 5436741f740SMichael Walsh 54481816748SMichael Walsh if transitional_boot_selected and not boot_success: 54581816748SMichael Walsh prior_boot = next_boot 54681816748SMichael Walsh boot_candidate = boot_stack.pop() 547004ad3c9SJoy Onyerikwu gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 548004ad3c9SJoy Onyerikwu + " transition to a valid state for '" + boot_candidate 549004ad3c9SJoy Onyerikwu + "' which was at the top of the boot_stack. Since" 550004ad3c9SJoy Onyerikwu + " the '" + next_boot + "' failed, the '" 551004ad3c9SJoy Onyerikwu + boot_candidate + "' has been removed from the stack" 552004ad3c9SJoy Onyerikwu + " to avoid and endless failure loop.") 55381816748SMichael Walsh if len(boot_stack) == 0: 55481816748SMichael Walsh return "" 55581816748SMichael Walsh 5566741f740SMichael Walsh my_get_state() 55745ca6e4cSMichael Walsh valid_state() 5586741f740SMichael Walsh 55981816748SMichael Walsh transitional_boot_selected = False 5606741f740SMichael Walsh stack_popped = 0 5616741f740SMichael Walsh if len(boot_stack) > 0: 5626741f740SMichael Walsh stack_popped = 1 563b5839d00SMichael Walsh gp.qprint_dashes() 564b5839d00SMichael Walsh gp.qprint_var(boot_stack) 565b5839d00SMichael Walsh gp.qprint_dashes() 566b5839d00SMichael Walsh skip_boot_printed = 0 567b5839d00SMichael Walsh while len(boot_stack) > 0: 5686741f740SMichael Walsh boot_candidate = boot_stack.pop() 569b5839d00SMichael Walsh if stack_mode == 'normal': 570b5839d00SMichael Walsh break 571b5839d00SMichael Walsh else: 572b5839d00SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['end']): 573b5839d00SMichael Walsh if not skip_boot_printed: 574ff340006SMichael Walsh gp.qprint_var(stack_mode) 575ff340006SMichael Walsh gp.qprintn() 576004ad3c9SJoy Onyerikwu gp.qprint_timen("Skipping the following boot tests" 577004ad3c9SJoy Onyerikwu + " which are unnecessary since their" 578004ad3c9SJoy Onyerikwu + " required end states match the" 579004ad3c9SJoy Onyerikwu + " current machine state:") 580b5839d00SMichael Walsh skip_boot_printed = 1 581ff340006SMichael Walsh gp.qprint_var(boot_candidate) 582b5839d00SMichael Walsh boot_candidate = "" 583b5839d00SMichael Walsh if boot_candidate == "": 584b5839d00SMichael Walsh gp.qprint_dashes() 585b5839d00SMichael Walsh gp.qprint_var(boot_stack) 586b5839d00SMichael Walsh gp.qprint_dashes() 587b5839d00SMichael Walsh return boot_candidate 5886741f740SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['start']): 589004ad3c9SJoy Onyerikwu gp.qprint_timen("The machine state is valid for a '" 590004ad3c9SJoy Onyerikwu + boot_candidate + "' boot test.") 591b5839d00SMichael Walsh gp.qprint_dashes() 592b5839d00SMichael Walsh gp.qprint_var(boot_stack) 593b5839d00SMichael Walsh gp.qprint_dashes() 5946741f740SMichael Walsh return boot_candidate 595341c21ebSMichael Walsh else: 596004ad3c9SJoy Onyerikwu gp.qprint_timen("The machine state does not match the required" 597004ad3c9SJoy Onyerikwu + " starting state for a '" + boot_candidate 598004ad3c9SJoy Onyerikwu + "' boot test:") 599ff340006SMichael Walsh gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 600b5839d00SMichael Walsh boot_table[boot_candidate]['start'], 1) 6016741f740SMichael Walsh boot_stack.append(boot_candidate) 60281816748SMichael Walsh transitional_boot_selected = True 6036741f740SMichael Walsh popped_boot = boot_candidate 6046741f740SMichael Walsh 6056741f740SMichael Walsh # Loop through your list selecting a boot_candidates 6066741f740SMichael Walsh boot_candidates = [] 6076741f740SMichael Walsh for boot_candidate in boot_list: 6086741f740SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['start']): 6096741f740SMichael Walsh if stack_popped: 6106741f740SMichael Walsh if st.compare_states(boot_table[boot_candidate]['end'], 6116741f740SMichael Walsh boot_table[popped_boot]['start']): 6126741f740SMichael Walsh boot_candidates.append(boot_candidate) 613341c21ebSMichael Walsh else: 6146741f740SMichael Walsh boot_candidates.append(boot_candidate) 6156741f740SMichael Walsh 6166741f740SMichael Walsh if len(boot_candidates) == 0: 617004ad3c9SJoy Onyerikwu gp.qprint_timen("The user's boot list contained no boot tests" 618004ad3c9SJoy Onyerikwu + " which are valid for the current machine state.") 6196741f740SMichael Walsh boot_candidate = default_power_on 6206741f740SMichael Walsh if not st.compare_states(state, boot_table[default_power_on]['start']): 6216741f740SMichael Walsh boot_candidate = default_power_off 6226741f740SMichael Walsh boot_candidates.append(boot_candidate) 623004ad3c9SJoy Onyerikwu gp.qprint_timen("Using default '" + boot_candidate 624004ad3c9SJoy Onyerikwu + "' boot type to transition to valid state.") 6256741f740SMichael Walsh 626b5839d00SMichael Walsh gp.dprint_var(boot_candidates) 6276741f740SMichael Walsh 6286741f740SMichael Walsh # Randomly select a boot from the candidate list. 6296741f740SMichael Walsh boot = random.choice(boot_candidates) 630341c21ebSMichael Walsh 631341c21ebSMichael Walsh return boot 6320bbd860fSMichael Walsh 63355302295SMichael Walsh 634341c21ebSMichael Walshdef print_last_boots(): 635341c21ebSMichael Walsh r""" 636341c21ebSMichael Walsh Print the last ten boots done with their time stamps. 637341c21ebSMichael Walsh """ 638341c21ebSMichael Walsh 639341c21ebSMichael Walsh # indent 0, 90 chars wide, linefeed, char is "=" 640b5839d00SMichael Walsh gp.qprint_dashes(0, 90) 641b5839d00SMichael Walsh gp.qprintn("Last 10 boots:\n") 642341c21ebSMichael Walsh 643341c21ebSMichael Walsh for boot_entry in last_ten: 644341c21ebSMichael Walsh grp.rqprint(boot_entry) 645b5839d00SMichael Walsh gp.qprint_dashes(0, 90) 646341c21ebSMichael Walsh 647341c21ebSMichael Walsh 648b2e53ecdSMichael Walshdef print_defect_report(ffdc_file_list): 649341c21ebSMichael Walsh r""" 650341c21ebSMichael Walsh Print a defect report. 651b2e53ecdSMichael Walsh 652b2e53ecdSMichael Walsh Description of argument(s): 653b2e53ecdSMichael Walsh ffdc_file_list A list of files which were collected by our ffdc functions. 654341c21ebSMichael Walsh """ 655341c21ebSMichael Walsh 656600876daSMichael Walsh # Making deliberate choice to NOT run plug_in_setup(). We don't want 657600876daSMichael Walsh # ffdc_prefix updated. 658600876daSMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 659600876daSMichael Walsh call_point='ffdc_report', stop_on_plug_in_failure=0) 660600876daSMichael Walsh 661e0cf8d70SMichael Walsh # Get additional header data which may have been created by ffdc plug-ins. 662e0cf8d70SMichael Walsh # Also, delete the individual header files to cleanup. 663e0cf8d70SMichael Walsh cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 664e0cf8d70SMichael Walsh " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 665e0cf8d70SMichael Walsh " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 666e0cf8d70SMichael Walsh shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 667e0cf8d70SMichael Walsh show_err=0) 668e0cf8d70SMichael Walsh 669b2e53ecdSMichael Walsh # Get additional summary data which may have been created by ffdc plug-ins. 670600876daSMichael Walsh # Also, delete the individual header files to cleanup. 671600876daSMichael Walsh cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 672600876daSMichael Walsh " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 673600876daSMichael Walsh " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 674600876daSMichael Walsh shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 675600876daSMichael Walsh show_err=0) 676600876daSMichael Walsh 677b2e53ecdSMichael Walsh # ffdc_list_file_path contains a list of any ffdc files created by plug- 678b2e53ecdSMichael Walsh # ins, etc. Read that data into a list. 679341c21ebSMichael Walsh try: 680b2e53ecdSMichael Walsh plug_in_ffdc_list = \ 681b2e53ecdSMichael Walsh open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 68236efbc04SGeorge Keishing plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 683341c21ebSMichael Walsh except IOError: 684b2e53ecdSMichael Walsh plug_in_ffdc_list = [] 685b2e53ecdSMichael Walsh 686b2e53ecdSMichael Walsh # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 687b2e53ecdSMichael Walsh # in. Eliminate duplicates and sort the list. 688004ad3c9SJoy Onyerikwu ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 689b2e53ecdSMichael Walsh 690b2e53ecdSMichael Walsh if status_file_path != "": 691b2e53ecdSMichael Walsh ffdc_file_list.insert(0, status_file_path) 692b2e53ecdSMichael Walsh 693b2e53ecdSMichael Walsh # Convert the list to a printable list. 694b2e53ecdSMichael Walsh printable_ffdc_file_list = "\n".join(ffdc_file_list) 695341c21ebSMichael Walsh 69668a61162SMichael Walsh # Open ffdc_file_list for writing. We will write a complete list of 69768a61162SMichael Walsh # FFDC files to it for possible use by plug-ins like cp_stop_check. 69868a61162SMichael Walsh ffdc_list_file = open(ffdc_list_file_path, 'w') 699b2e53ecdSMichael Walsh ffdc_list_file.write(printable_ffdc_file_list + "\n") 700b2e53ecdSMichael Walsh ffdc_list_file.close() 701b2e53ecdSMichael Walsh 702b2e53ecdSMichael Walsh indent = 0 703b2e53ecdSMichael Walsh width = 90 704b2e53ecdSMichael Walsh linefeed = 1 705b2e53ecdSMichael Walsh char = "=" 70668a61162SMichael Walsh 70768a61162SMichael Walsh gp.qprintn() 708b2e53ecdSMichael Walsh gp.qprint_dashes(indent, width, linefeed, char) 70968a61162SMichael Walsh gp.qprintn("Copy this data to the defect:\n") 71068a61162SMichael Walsh 711e0cf8d70SMichael Walsh if len(more_header_info) > 0: 712ff340006SMichael Walsh gp.qprintn(more_header_info) 713dc80d67dSMichael Walsh gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 714dc80d67dSMichael Walsh openbmc_host_name, openbmc_ip, openbmc_username, 7150a3bdb4cSMichael Walsh openbmc_password, rest_username, rest_password, ipmi_username, 7160a3bdb4cSMichael Walsh ipmi_password, os_host, os_host_name, os_ip, os_username, 717dc80d67dSMichael Walsh os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 718dc80d67dSMichael Walsh pdu_password, pdu_slot_no, openbmc_serial_host, 719dc80d67dSMichael Walsh openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 72068a61162SMichael Walsh 72168a61162SMichael Walsh gp.qprintn() 72268a61162SMichael Walsh print_last_boots() 72368a61162SMichael Walsh gp.qprintn() 72468a61162SMichael Walsh gp.qprint_var(state) 725b5839d00SMichael Walsh gp.qprintn() 726b5839d00SMichael Walsh gp.qprintn("FFDC data files:") 727b2e53ecdSMichael Walsh gp.qprintn(printable_ffdc_file_list) 728b5839d00SMichael Walsh gp.qprintn() 729341c21ebSMichael Walsh 730600876daSMichael Walsh if len(ffdc_summary_info) > 0: 731ff340006SMichael Walsh gp.qprintn(ffdc_summary_info) 732600876daSMichael Walsh 733b2e53ecdSMichael Walsh gp.qprint_dashes(indent, width, linefeed, char) 73468a61162SMichael Walsh 7356741f740SMichael Walsh 7366741f740SMichael Walshdef my_ffdc(): 7376741f740SMichael Walsh r""" 7386741f740SMichael Walsh Collect FFDC data. 7396741f740SMichael Walsh """ 7406741f740SMichael Walsh 7416741f740SMichael Walsh global state 7426741f740SMichael Walsh 7436741f740SMichael Walsh plug_in_setup() 7446741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 745600876daSMichael Walsh call_point='ffdc', stop_on_plug_in_failure=0) 7466741f740SMichael Walsh 7476741f740SMichael Walsh AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 748004ad3c9SJoy Onyerikwu status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 749004ad3c9SJoy Onyerikwu + AUTOBOOT_FFDC_PREFIX 750004ad3c9SJoy Onyerikwu + " ffdc_function_list=" 751004ad3c9SJoy Onyerikwu + ffdc_function_list, ignore=1) 75283f4bc77SMichael Walsh if status != 'PASS': 753ff340006SMichael Walsh gp.qprint_error("Call to ffdc failed.\n") 7546741f740SMichael Walsh 7556741f740SMichael Walsh my_get_state() 7566741f740SMichael Walsh 757b2e53ecdSMichael Walsh print_defect_report(ffdc_file_list) 7586741f740SMichael Walsh 7596741f740SMichael Walsh 7606741f740SMichael Walshdef print_test_start_message(boot_keyword): 7616741f740SMichael Walsh r""" 7626741f740SMichael Walsh Print a message indicating what boot test is about to run. 7636741f740SMichael Walsh 7646741f740SMichael Walsh Description of arguments: 7656741f740SMichael Walsh boot_keyword The name of the boot which is to be run 7666741f740SMichael Walsh (e.g. "BMC Power On"). 7676741f740SMichael Walsh """ 7686741f740SMichael Walsh 7696741f740SMichael Walsh global last_ten 770325eb548SSunil M global boot_start_time 7716741f740SMichael Walsh 7726741f740SMichael Walsh doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 773325eb548SSunil M 774325eb548SSunil M # Set boot_start_time for use by plug-ins. 775325eb548SSunil M boot_start_time = doing_msg[1:33] 776325eb548SSunil M gp.qprint_var(boot_start_time) 777325eb548SSunil M 778b5839d00SMichael Walsh gp.qprint(doing_msg) 7796741f740SMichael Walsh 7806741f740SMichael Walsh last_ten.append(doing_msg) 7816741f740SMichael Walsh 782815b1d5bSMichael Walsh # Trim list to max number of entries. 783815b1d5bSMichael Walsh del last_ten[:max(0, len(last_ten) - max_boot_history)] 7846741f740SMichael Walsh 7856741f740SMichael Walsh 786f566fb1fSMichael Walshdef stop_boot_test(signal_number=0, 787f566fb1fSMichael Walsh frame=None): 788f566fb1fSMichael Walsh r""" 789f566fb1fSMichael Walsh Handle SIGUSR1 by aborting the boot test that is running. 790f566fb1fSMichael Walsh 791f566fb1fSMichael Walsh Description of argument(s): 792f566fb1fSMichael Walsh signal_number The signal number (should always be 10 for SIGUSR1). 793f566fb1fSMichael Walsh frame The frame data. 794f566fb1fSMichael Walsh """ 795f566fb1fSMichael Walsh 796f566fb1fSMichael Walsh gp.printn() 797f566fb1fSMichael Walsh gp.print_executing() 798f566fb1fSMichael Walsh gp.lprint_executing() 799f566fb1fSMichael Walsh 800f566fb1fSMichael Walsh # Restore original sigusr1 handler. 801f566fb1fSMichael Walsh set_default_siguser1() 802f566fb1fSMichael Walsh 803f566fb1fSMichael Walsh message = "The caller has asked that the boot test be stopped and marked" 804f566fb1fSMichael Walsh message += " as a failure." 805f566fb1fSMichael Walsh 806f566fb1fSMichael Walsh function_stack = gm.get_function_stack() 807f566fb1fSMichael Walsh if "wait_state" in function_stack: 808f566fb1fSMichael Walsh st.set_wait_early_exit_message(message) 809f566fb1fSMichael Walsh else: 810f566fb1fSMichael Walsh BuiltIn().fail(gp.sprint_error(message)) 811f566fb1fSMichael Walsh 812f566fb1fSMichael Walsh 8136741f740SMichael Walshdef run_boot(boot): 8146741f740SMichael Walsh r""" 8156741f740SMichael Walsh Run the specified boot. 8166741f740SMichael Walsh 8176741f740SMichael Walsh Description of arguments: 8186741f740SMichael Walsh boot The name of the boot test to be performed. 8196741f740SMichael Walsh """ 8206741f740SMichael Walsh 8216741f740SMichael Walsh global state 8226741f740SMichael Walsh 823f566fb1fSMichael Walsh signal.signal(signal.SIGUSR1, stop_boot_test) 824f566fb1fSMichael Walsh gp.qprint_timen("stop_boot_test is armed.") 825f566fb1fSMichael Walsh 8266741f740SMichael Walsh print_test_start_message(boot) 8276741f740SMichael Walsh 8286741f740SMichael Walsh plug_in_setup() 8296741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 8306741f740SMichael Walsh grpi.rprocess_plug_in_packages(call_point="pre_boot") 8316741f740SMichael Walsh if rc != 0: 8326741f740SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" +\ 8336741f740SMichael Walsh gp.sprint_var(rc, 1) 834f566fb1fSMichael Walsh set_default_siguser1() 8356741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 8366741f740SMichael Walsh 8376741f740SMichael Walsh if test_mode: 8386741f740SMichael Walsh # In test mode, we'll pretend the boot worked by assigning its 8396741f740SMichael Walsh # required end state to the default state value. 84030dadae2SMichael Walsh state = st.strip_anchor_state(boot_table[boot]['end']) 8416741f740SMichael Walsh else: 8426741f740SMichael Walsh # Assertion: We trust that the state data was made fresh by the 8436741f740SMichael Walsh # caller. 8446741f740SMichael Walsh 845b5839d00SMichael Walsh gp.qprintn() 8466741f740SMichael Walsh 8476741f740SMichael Walsh if boot_table[boot]['method_type'] == "keyword": 8480b93fbf8SMichael Walsh rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 849b5839d00SMichael Walsh boot_table[boot]['method'], 850b5839d00SMichael Walsh quiet=quiet) 8516741f740SMichael Walsh 8526741f740SMichael Walsh if boot_table[boot]['bmc_reboot']: 8536741f740SMichael Walsh st.wait_for_comm_cycle(int(state['epoch_seconds'])) 85430dadae2SMichael Walsh plug_in_setup() 85530dadae2SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 85630dadae2SMichael Walsh grpi.rprocess_plug_in_packages(call_point="post_reboot") 85730dadae2SMichael Walsh if rc != 0: 8580b93fbf8SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" 8590b93fbf8SMichael Walsh error_message += gp.sprint_var(rc, 1) 860f566fb1fSMichael Walsh set_default_siguser1() 86130dadae2SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 8626741f740SMichael Walsh else: 8636741f740SMichael Walsh match_state = st.anchor_state(state) 8646741f740SMichael Walsh del match_state['epoch_seconds'] 8656741f740SMichael Walsh # Wait for the state to change in any way. 8666741f740SMichael Walsh st.wait_state(match_state, wait_time=state_change_timeout, 867600876daSMichael Walsh interval="10 seconds", invert=1) 8686741f740SMichael Walsh 869b5839d00SMichael Walsh gp.qprintn() 8706741f740SMichael Walsh if boot_table[boot]['end']['chassis'] == "Off": 8716741f740SMichael Walsh boot_timeout = power_off_timeout 8726741f740SMichael Walsh else: 8736741f740SMichael Walsh boot_timeout = power_on_timeout 8746741f740SMichael Walsh st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 875600876daSMichael Walsh interval="10 seconds") 8766741f740SMichael Walsh 8776741f740SMichael Walsh plug_in_setup() 8786741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 8796741f740SMichael Walsh grpi.rprocess_plug_in_packages(call_point="post_boot") 8806741f740SMichael Walsh if rc != 0: 8816741f740SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" +\ 8826741f740SMichael Walsh gp.sprint_var(rc, 1) 883f566fb1fSMichael Walsh set_default_siguser1() 8846741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 8856741f740SMichael Walsh 886f566fb1fSMichael Walsh # Restore original sigusr1 handler. 887f566fb1fSMichael Walsh set_default_siguser1() 888f566fb1fSMichael Walsh 8896741f740SMichael Walsh 8906741f740SMichael Walshdef test_loop_body(): 8916741f740SMichael Walsh r""" 8926741f740SMichael Walsh The main loop body for the loop in main_py. 8936741f740SMichael Walsh 8946741f740SMichael Walsh Description of arguments: 8956741f740SMichael Walsh boot_count The iteration number (starts at 1). 8966741f740SMichael Walsh """ 8976741f740SMichael Walsh 8986741f740SMichael Walsh global boot_count 8996741f740SMichael Walsh global state 9006741f740SMichael Walsh global next_boot 9016741f740SMichael Walsh global boot_success 902325eb548SSunil M global boot_end_time 9036741f740SMichael Walsh 904b5839d00SMichael Walsh gp.qprintn() 9056741f740SMichael Walsh 9066741f740SMichael Walsh next_boot = select_boot() 907b5839d00SMichael Walsh if next_boot == "": 908b5839d00SMichael Walsh return True 9096741f740SMichael Walsh 910b5839d00SMichael Walsh boot_count += 1 911b5839d00SMichael Walsh gp.qprint_timen("Starting boot " + str(boot_count) + ".") 9126741f740SMichael Walsh 913e0cf8d70SMichael Walsh pre_boot_plug_in_setup() 9146741f740SMichael Walsh 9156741f740SMichael Walsh cmd_buf = ["run_boot", next_boot] 9166741f740SMichael Walsh boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 9176741f740SMichael Walsh if boot_status == "FAIL": 918b5839d00SMichael Walsh gp.qprint(msg) 9196741f740SMichael Walsh 920b5839d00SMichael Walsh gp.qprintn() 9216741f740SMichael Walsh if boot_status == "PASS": 9226741f740SMichael Walsh boot_success = 1 923004ad3c9SJoy Onyerikwu completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 924004ad3c9SJoy Onyerikwu + "\" succeeded.") 9256741f740SMichael Walsh else: 9266741f740SMichael Walsh boot_success = 0 927004ad3c9SJoy Onyerikwu completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 928004ad3c9SJoy Onyerikwu + "\" failed.") 929325eb548SSunil M 930325eb548SSunil M # Set boot_end_time for use by plug-ins. 931325eb548SSunil M boot_end_time = completion_msg[1:33] 932325eb548SSunil M gp.qprint_var(boot_end_time) 933325eb548SSunil M 934325eb548SSunil M gp.qprint(completion_msg) 9356741f740SMichael Walsh 9366741f740SMichael Walsh boot_results.update(next_boot, boot_status) 9376741f740SMichael Walsh 9386741f740SMichael Walsh plug_in_setup() 9396741f740SMichael Walsh # NOTE: A post_test_case call point failure is NOT counted as a boot 9406741f740SMichael Walsh # failure. 9416741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 942600876daSMichael Walsh call_point='post_test_case', stop_on_plug_in_failure=0) 9436741f740SMichael Walsh 9446741f740SMichael Walsh plug_in_setup() 9456741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 94689de14a4SMichael Walsh call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 9476741f740SMichael Walsh stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 948*12059e26SMichael Walsh if ffdc_check == "All" or\ 94989de14a4SMichael Walsh shell_rc == dump_ffdc_rc(): 95083f4bc77SMichael Walsh status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 95183f4bc77SMichael Walsh if status != 'PASS': 952ff340006SMichael Walsh gp.qprint_error("Call to my_ffdc failed.\n") 9536741f740SMichael Walsh 954aabef1e3SMichael Walsh if delete_errlogs: 955d139f286SMichael Walsh # We need to purge error logs between boots or they build up. 956b5839d00SMichael Walsh grk.run_key("Delete Error logs", ignore=1) 957d139f286SMichael Walsh 958952f9b09SMichael Walsh boot_results.print_report() 959b5839d00SMichael Walsh gp.qprint_timen("Finished boot " + str(boot_count) + ".") 960952f9b09SMichael Walsh 9616741f740SMichael Walsh plug_in_setup() 9626741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 96389de14a4SMichael Walsh call_point='stop_check', shell_rc=stop_test_rc(), 96489de14a4SMichael Walsh stop_on_non_zero_rc=1) 96589de14a4SMichael Walsh if shell_rc == stop_test_rc(): 9663ba8ecdcSMichael Walsh message = "Stopping as requested by user.\n" 9673ba8ecdcSMichael Walsh gp.print_time(message) 9683ba8ecdcSMichael Walsh BuiltIn().fail(message) 9696741f740SMichael Walsh 970d139f286SMichael Walsh # This should help prevent ConnectionErrors. 9710960b384SMichael Walsh grk.run_key_u("Close All Connections") 972d139f286SMichael Walsh 9736741f740SMichael Walsh return True 9746741f740SMichael Walsh 9756741f740SMichael Walsh 97683f4bc77SMichael Walshdef obmc_boot_test_teardown(): 9776741f740SMichael Walsh r""" 978c9116811SMichael Walsh Clean up after the Main keyword. 9796741f740SMichael Walsh """ 9806741f740SMichael Walsh 9816741f740SMichael Walsh if cp_setup_called: 9826741f740SMichael Walsh plug_in_setup() 9836741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 984600876daSMichael Walsh call_point='cleanup', stop_on_plug_in_failure=0) 9856741f740SMichael Walsh 986600876daSMichael Walsh if 'boot_results_file_path' in globals(): 9876c64574bSMichael Walsh # Save boot_results and last_ten objects to a file in case they are 9886c64574bSMichael Walsh # needed again. 989b5839d00SMichael Walsh gp.qprint_timen("Saving boot_results to the following path.") 990b5839d00SMichael Walsh gp.qprint_var(boot_results_file_path) 9916c64574bSMichael Walsh pickle.dump((boot_results, last_ten), 9926c64574bSMichael Walsh open(boot_results_file_path, 'wb'), 9930b93fbf8SMichael Walsh pickle.HIGHEST_PROTOCOL) 9940b93fbf8SMichael Walsh 995ff340006SMichael Walsh global save_stack 996ff340006SMichael Walsh # Restore any global values saved on the save_stack. 997ff340006SMichael Walsh for parm_name in main_func_parm_list: 998ff340006SMichael Walsh # Get the parm_value if it was saved on the stack. 999ff340006SMichael Walsh try: 1000ff340006SMichael Walsh parm_value = save_stack.pop(parm_name) 1001004ad3c9SJoy Onyerikwu except BaseException: 1002ff340006SMichael Walsh # If it was not saved, no further action is required. 1003ff340006SMichael Walsh continue 1004ff340006SMichael Walsh 1005ff340006SMichael Walsh # Restore the saved value. 1006ff340006SMichael Walsh cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1007ff340006SMichael Walsh "}\", parm_value)" 1008ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1009ff340006SMichael Walsh exec(cmd_buf) 1010ff340006SMichael Walsh 1011ff340006SMichael Walsh gp.dprintn(save_stack.sprint_obj()) 1012ff340006SMichael Walsh 10136741f740SMichael Walsh 1014c9116811SMichael Walshdef test_teardown(): 1015c9116811SMichael Walsh r""" 1016c9116811SMichael Walsh Clean up after this test case. 1017c9116811SMichael Walsh """ 1018c9116811SMichael Walsh 1019c9116811SMichael Walsh gp.qprintn() 1020c9116811SMichael Walsh cmd_buf = ["Print Error", 1021c9116811SMichael Walsh "A keyword timeout occurred ending this program.\n"] 1022c9116811SMichael Walsh BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1023c9116811SMichael Walsh 1024b5839d00SMichael Walsh grp.rqprint_pgm_footer() 1025b5839d00SMichael Walsh 1026c9116811SMichael Walsh 102789de14a4SMichael Walshdef post_stack(): 102889de14a4SMichael Walsh r""" 102989de14a4SMichael Walsh Process post_stack plug-in programs. 103089de14a4SMichael Walsh """ 103189de14a4SMichael Walsh 103289de14a4SMichael Walsh if not call_post_stack_plug: 103389de14a4SMichael Walsh # The caller does not wish to have post_stack plug-in processing done. 103489de14a4SMichael Walsh return 103589de14a4SMichael Walsh 103689de14a4SMichael Walsh global boot_success 103789de14a4SMichael Walsh 103889de14a4SMichael Walsh # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 103989de14a4SMichael Walsh pre_boot_plug_in_setup() 104089de14a4SMichael Walsh # For the purposes of the following plug-ins, mark the "boot" as a success. 104189de14a4SMichael Walsh boot_success = 1 104289de14a4SMichael Walsh plug_in_setup() 1043815b1d5bSMichael Walsh rc, shell_rc, failed_plug_in_name, history =\ 1044815b1d5bSMichael Walsh grpi.rprocess_plug_in_packages(call_point='post_stack', 1045815b1d5bSMichael Walsh stop_on_plug_in_failure=0, 1046815b1d5bSMichael Walsh return_history=True) 1047815b1d5bSMichael Walsh last_ten.extend(history) 1048815b1d5bSMichael Walsh # Trim list to max number of entries. 1049815b1d5bSMichael Walsh del last_ten[:max(0, len(last_ten) - max_boot_history)] 1050815b1d5bSMichael Walsh if rc != 0: 1051815b1d5bSMichael Walsh boot_success = 0 105289de14a4SMichael Walsh 105389de14a4SMichael Walsh plug_in_setup() 1054815b1d5bSMichael Walsh rc, shell_rc, failed_plug_in_name =\ 1055815b1d5bSMichael Walsh grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1056815b1d5bSMichael Walsh shell_rc=dump_ffdc_rc(), 1057815b1d5bSMichael Walsh stop_on_plug_in_failure=1, 1058815b1d5bSMichael Walsh stop_on_non_zero_rc=1) 1059815b1d5bSMichael Walsh if shell_rc == dump_ffdc_rc(): 106089de14a4SMichael Walsh status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 106189de14a4SMichael Walsh if status != 'PASS': 106289de14a4SMichael Walsh gp.qprint_error("Call to my_ffdc failed.\n") 106389de14a4SMichael Walsh 106489de14a4SMichael Walsh plug_in_setup() 106589de14a4SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 106689de14a4SMichael Walsh call_point='stop_check', shell_rc=stop_test_rc(), 106789de14a4SMichael Walsh stop_on_non_zero_rc=1) 106889de14a4SMichael Walsh if shell_rc == stop_test_rc(): 106989de14a4SMichael Walsh message = "Stopping as requested by user.\n" 107089de14a4SMichael Walsh gp.print_time(message) 107189de14a4SMichael Walsh BuiltIn().fail(message) 107289de14a4SMichael Walsh 107389de14a4SMichael Walsh 1074ff340006SMichael Walshdef obmc_boot_test_py(loc_boot_stack=None, 1075ff340006SMichael Walsh loc_stack_mode=None, 1076ff340006SMichael Walsh loc_quiet=None): 10776741f740SMichael Walsh r""" 10786741f740SMichael Walsh Do main program processing. 10796741f740SMichael Walsh """ 10806741f740SMichael Walsh 1081ff340006SMichael Walsh global save_stack 1082ff340006SMichael Walsh 108336efbc04SGeorge Keishing gp.dprintn() 1084ff340006SMichael Walsh # Process function parms. 1085ff340006SMichael Walsh for parm_name in main_func_parm_list: 1086ff340006SMichael Walsh # Get parm's value. 108736efbc04SGeorge Keishing parm_value = eval("loc_" + parm_name) 108836efbc04SGeorge Keishing gp.dpvars(parm_name, parm_value) 1089ff340006SMichael Walsh 109036efbc04SGeorge Keishing if parm_value is not None: 1091ff340006SMichael Walsh # Save the global value on a stack. 1092ff340006SMichael Walsh cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1093ff340006SMichael Walsh parm_name + "}\"), \"" + parm_name + "\")" 1094ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1095ff340006SMichael Walsh exec(cmd_buf) 1096ff340006SMichael Walsh 1097ff340006SMichael Walsh # Set the global value to the passed value. 1098ff340006SMichael Walsh cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1099ff340006SMichael Walsh "}\", loc_" + parm_name + ")" 1100ff340006SMichael Walsh gp.dpissuing(cmd_buf) 1101ff340006SMichael Walsh exec(cmd_buf) 1102ff340006SMichael Walsh 1103ff340006SMichael Walsh gp.dprintn(save_stack.sprint_obj()) 1104b5839d00SMichael Walsh 11056741f740SMichael Walsh setup() 11066741f740SMichael Walsh 1107cd9fbfd7SMichael Walsh init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1108cd9fbfd7SMichael Walsh 1109a20da401SMichael Walsh if ffdc_only: 1110a20da401SMichael Walsh gp.qprint_timen("Caller requested ffdc_only.") 1111e0cf8d70SMichael Walsh pre_boot_plug_in_setup() 111283f4bc77SMichael Walsh grk.run_key_u("my_ffdc") 1113764d2f83SMichael Walsh return 1114a20da401SMichael Walsh 11156741f740SMichael Walsh # Process caller's boot_stack. 11166741f740SMichael Walsh while (len(boot_stack) > 0): 11176741f740SMichael Walsh test_loop_body() 11186741f740SMichael Walsh 1119b5839d00SMichael Walsh gp.qprint_timen("Finished processing stack.") 112030dadae2SMichael Walsh 112189de14a4SMichael Walsh post_stack() 112289de14a4SMichael Walsh 11236741f740SMichael Walsh # Process caller's boot_list. 11246741f740SMichael Walsh if len(boot_list) > 0: 11256741f740SMichael Walsh for ix in range(1, max_num_tests + 1): 11266741f740SMichael Walsh test_loop_body() 11276741f740SMichael Walsh 1128b5839d00SMichael Walsh gp.qprint_timen("Completed all requested boot tests.") 1129b5839d00SMichael Walsh 1130b5839d00SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 1131cd9fbfd7SMichael Walsh new_fail = boot_fail - init_boot_fail 1132cd9fbfd7SMichael Walsh if new_fail > boot_fail_threshold: 1133b5839d00SMichael Walsh error_message = "Boot failures exceed the boot failure" +\ 1134b5839d00SMichael Walsh " threshold:\n" +\ 1135cd9fbfd7SMichael Walsh gp.sprint_var(new_fail) +\ 1136b5839d00SMichael Walsh gp.sprint_var(boot_fail_threshold) 1137b5839d00SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 1138