10bbd860fSMichael Walsh#!/usr/bin/env python 20bbd860fSMichael Walsh 30bbd860fSMichael Walshr""" 40bbd860fSMichael WalshThis module is the python counterpart to obmc_boot_test. 50bbd860fSMichael Walsh""" 60bbd860fSMichael Walsh 70b93fbf8SMichael Walshimport os 80b93fbf8SMichael Walshimport imp 90b93fbf8SMichael Walshimport time 100b93fbf8SMichael Walshimport glob 110b93fbf8SMichael Walshimport random 120b93fbf8SMichael Walshimport cPickle as pickle 130b93fbf8SMichael Walsh 140b93fbf8SMichael Walshfrom robot.utils import DotDict 150b93fbf8SMichael Walshfrom robot.libraries.BuiltIn import BuiltIn 160b93fbf8SMichael Walsh 176741f740SMichael Walshfrom boot_data import * 18c9116811SMichael Walshimport gen_print as gp 190bbd860fSMichael Walshimport gen_robot_print as grp 2055302295SMichael Walshimport gen_robot_plug_in as grpi 216741f740SMichael Walshimport gen_robot_valid as grv 226741f740SMichael Walshimport gen_misc as gm 236741f740SMichael Walshimport gen_cmd as gc 24b5839d00SMichael Walshimport gen_robot_keyword as grk 2555302295SMichael Walshimport state as st 260bbd860fSMichael Walsh 270b93fbf8SMichael Walshbase_path = os.path.dirname(os.path.dirname( 280b93fbf8SMichael Walsh imp.find_module("gen_robot_print")[1])) +\ 290b93fbf8SMichael Walsh os.sep 300b93fbf8SMichael Walshsys.path.append(base_path + "extended/") 310b93fbf8SMichael Walshimport run_keyword as rk 320bbd860fSMichael Walsh 33e1e26448SMichael Walsh# Setting master_pid correctly influences the behavior of plug-ins like 34e1e26448SMichael Walsh# DB_Logging 35e1e26448SMichael Walshprogram_pid = os.getpid() 36e1e26448SMichael Walshmaster_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 37e1e26448SMichael Walsh 38b5839d00SMichael Walsh# Set up boot data structures. 39b5839d00SMichael Walshboot_table = create_boot_table() 40b5839d00SMichael Walshvalid_boot_types = create_valid_boot_list(boot_table) 410b93fbf8SMichael Walsh 426741f740SMichael Walshboot_lists = read_boot_lists() 436741f740SMichael Walshlast_ten = [] 446741f740SMichael Walsh 456741f740SMichael Walshstate = st.return_default_state() 466741f740SMichael Walshcp_setup_called = 0 476741f740SMichael Walshnext_boot = "" 486741f740SMichael Walshbase_tool_dir_path = os.path.normpath(os.environ.get( 496741f740SMichael Walsh 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 50b5839d00SMichael Walsh 516741f740SMichael Walshffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 526741f740SMichael Walshboot_success = 0 536741f740SMichael Walshstatus_dir_path = os.environ.get('STATUS_DIR_PATH', "") 546741f740SMichael Walshif status_dir_path != "": 556741f740SMichael Walsh status_dir_path = os.path.normpath(status_dir_path) + os.sep 560b93fbf8SMichael Walshdefault_power_on = "REST Power On" 570b93fbf8SMichael Walshdefault_power_off = "REST Power Off" 586741f740SMichael Walshboot_count = 0 590bbd860fSMichael Walsh 6085678948SMichael WalshLOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 6185678948SMichael Walsh 6285678948SMichael Walsh 6385678948SMichael Walsh############################################################################### 64b5839d00SMichael Walshdef process_pgm_parms(): 65b5839d00SMichael Walsh 66b5839d00SMichael Walsh r""" 67b5839d00SMichael Walsh Process the program parameters by assigning them all to corresponding 68b5839d00SMichael Walsh globals. Also, set some global values that depend on program parameters. 69b5839d00SMichael Walsh """ 70b5839d00SMichael Walsh 71b5839d00SMichael Walsh # Program parameter processing. 72b5839d00SMichael Walsh # Assign all program parms to python variables which are global to this 73b5839d00SMichael Walsh # module. 74b5839d00SMichael Walsh 75b5839d00SMichael Walsh global parm_list 76b5839d00SMichael Walsh parm_list = BuiltIn().get_variable_value("${parm_list}") 77b5839d00SMichael Walsh # The following subset of parms should be processed as integers. 78b5839d00SMichael Walsh int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 79b5839d00SMichael Walsh 'boot_fail_threshold', 'quiet', 'test_mode', 'debug'] 80b5839d00SMichael Walsh for parm in parm_list: 81b5839d00SMichael Walsh if parm in int_list: 82b5839d00SMichael Walsh sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 83b5839d00SMichael Walsh "}\", \"0\"))" 84b5839d00SMichael Walsh else: 85b5839d00SMichael Walsh sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 86b5839d00SMichael Walsh cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 87b5839d00SMichael Walsh exec(cmd_buf) 88b5839d00SMichael Walsh 89b5839d00SMichael Walsh global ffdc_dir_path_style 90b5839d00SMichael Walsh global boot_list 91b5839d00SMichael Walsh global boot_stack 92b5839d00SMichael Walsh global boot_results_file_path 93b5839d00SMichael Walsh global boot_results 94b5839d00SMichael Walsh global ffdc_list_file_path 95b5839d00SMichael Walsh 96b5839d00SMichael Walsh if ffdc_dir_path_style == "": 97b5839d00SMichael Walsh ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 98b5839d00SMichael Walsh 99b5839d00SMichael Walsh # Convert these program parms to lists for easier processing.. 100b5839d00SMichael Walsh boot_list = filter(None, boot_list.split(":")) 101b5839d00SMichael Walsh boot_stack = filter(None, boot_stack.split(":")) 102b5839d00SMichael Walsh 103b5839d00SMichael Walsh boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\ 104b5839d00SMichael Walsh str(master_pid) + ":boot_results" 105b5839d00SMichael Walsh 106b5839d00SMichael Walsh if os.path.isfile(boot_results_file_path): 107b5839d00SMichael Walsh # We've been called before in this run so we'll load the saved 108b5839d00SMichael Walsh # boot_results object. 109b5839d00SMichael Walsh boot_results = pickle.load(open(boot_results_file_path, 'rb')) 110b5839d00SMichael Walsh else: 111b5839d00SMichael Walsh boot_results = boot_results(boot_table, boot_pass, boot_fail) 112b5839d00SMichael Walsh 113b5839d00SMichael Walsh ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 114b5839d00SMichael Walsh "/FFDC_FILE_LIST" 115b5839d00SMichael Walsh 116b5839d00SMichael Walsh############################################################################### 117b5839d00SMichael Walsh 118b5839d00SMichael Walsh 119b5839d00SMichael Walsh############################################################################### 12085678948SMichael Walshdef initial_plug_in_setup(): 12185678948SMichael Walsh 12285678948SMichael Walsh r""" 12385678948SMichael Walsh Initialize all plug-in environment variables which do not change for the 12485678948SMichael Walsh duration of the program. 12585678948SMichael Walsh 12685678948SMichael Walsh """ 12785678948SMichael Walsh 12885678948SMichael Walsh global LOG_LEVEL 12985678948SMichael Walsh BuiltIn().set_log_level("NONE") 13085678948SMichael Walsh 13185678948SMichael Walsh BuiltIn().set_global_variable("${master_pid}", master_pid) 13285678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 13385678948SMichael Walsh BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 13485678948SMichael Walsh BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 13585678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 13685678948SMichael Walsh ffdc_list_file_path) 13785678948SMichael Walsh 13885678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 13985678948SMichael Walsh ffdc_dir_path_style) 14085678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_CHECK}", 14185678948SMichael Walsh ffdc_check) 14285678948SMichael Walsh 14385678948SMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 14485678948SMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 14585678948SMichael Walsh # element in additional_values. 14685678948SMichael Walsh additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 14785678948SMichael Walsh "status_dir_path", "base_tool_dir_path", 14885678948SMichael Walsh "ffdc_list_file_path"] 14985678948SMichael Walsh 15085678948SMichael Walsh plug_in_vars = parm_list + additional_values 15185678948SMichael Walsh 15285678948SMichael Walsh for var_name in plug_in_vars: 15385678948SMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 15485678948SMichael Walsh var_name = var_name.upper() 15585678948SMichael Walsh if var_value is None: 15685678948SMichael Walsh var_value = "" 15785678948SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 15885678948SMichael Walsh 15985678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 16085678948SMichael Walsh 16185678948SMichael Walsh 16285678948SMichael Walsh############################################################################### 16385678948SMichael Walsh 1640bbd860fSMichael Walsh 1650bbd860fSMichael Walsh############################################################################### 1660bbd860fSMichael Walshdef plug_in_setup(): 1670bbd860fSMichael Walsh 1680bbd860fSMichael Walsh r""" 16985678948SMichael Walsh Initialize all changing plug-in environment variables for use by the 17085678948SMichael Walsh plug-in programs. 1710bbd860fSMichael Walsh """ 1720bbd860fSMichael Walsh 17385678948SMichael Walsh global LOG_LEVEL 17485678948SMichael Walsh global test_really_running 17585678948SMichael Walsh 17685678948SMichael Walsh BuiltIn().set_log_level("NONE") 17785678948SMichael Walsh 1786741f740SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 1790bbd860fSMichael Walsh if boot_pass > 1: 1800bbd860fSMichael Walsh test_really_running = 1 1810bbd860fSMichael Walsh else: 1820bbd860fSMichael Walsh test_really_running = 0 1830bbd860fSMichael Walsh 1840bbd860fSMichael Walsh seconds = time.time() 1850bbd860fSMichael Walsh loc_time = time.localtime(seconds) 1860bbd860fSMichael Walsh time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 1870bbd860fSMichael Walsh 1886741f740SMichael Walsh ffdc_prefix = openbmc_nickname + "." + time_string 1890bbd860fSMichael Walsh 1906741f740SMichael Walsh BuiltIn().set_global_variable("${test_really_running}", 1916741f740SMichael Walsh test_really_running) 1926741f740SMichael Walsh BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 1936741f740SMichael Walsh BuiltIn().set_global_variable("${boot_pass}", boot_pass) 1946741f740SMichael Walsh BuiltIn().set_global_variable("${boot_fail}", boot_fail) 1956741f740SMichael Walsh BuiltIn().set_global_variable("${boot_success}", boot_success) 1966741f740SMichael Walsh BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 1974c9a6453SMichael Walsh 1980bbd860fSMichael Walsh # For each program parameter, set the corresponding AUTOBOOT_ environment 1990bbd860fSMichael Walsh # variable value. Also, set an AUTOBOOT_ environment variable for every 2000bbd860fSMichael Walsh # element in additional_values. 2010bbd860fSMichael Walsh additional_values = ["boot_type_desc", "boot_success", "boot_pass", 20285678948SMichael Walsh "boot_fail", "test_really_running", "ffdc_prefix"] 2030bbd860fSMichael Walsh 20485678948SMichael Walsh plug_in_vars = additional_values 2050bbd860fSMichael Walsh 2060bbd860fSMichael Walsh for var_name in plug_in_vars: 2070bbd860fSMichael Walsh var_value = BuiltIn().get_variable_value("${" + var_name + "}") 2080bbd860fSMichael Walsh var_name = var_name.upper() 2090bbd860fSMichael Walsh if var_value is None: 2100bbd860fSMichael Walsh var_value = "" 2116741f740SMichael Walsh os.environ["AUTOBOOT_" + var_name] = str(var_value) 2120bbd860fSMichael Walsh 2130bbd860fSMichael Walsh if debug: 2146741f740SMichael Walsh shell_rc, out_buf = \ 2156741f740SMichael Walsh gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 2160bbd860fSMichael Walsh 21785678948SMichael Walsh BuiltIn().set_log_level(LOG_LEVEL) 21885678948SMichael Walsh 2190bbd860fSMichael Walsh############################################################################### 2200bbd860fSMichael Walsh 2210bbd860fSMichael Walsh 2220bbd860fSMichael Walsh############################################################################### 2236741f740SMichael Walshdef setup(): 2240bbd860fSMichael Walsh 2250bbd860fSMichael Walsh r""" 2266741f740SMichael Walsh Do general program setup tasks. 2270bbd860fSMichael Walsh """ 2280bbd860fSMichael Walsh 2296741f740SMichael Walsh global cp_setup_called 2300bbd860fSMichael Walsh 231b5839d00SMichael Walsh gp.qprintn() 232b5839d00SMichael Walsh 233*83f4bc77SMichael Walsh robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 234*83f4bc77SMichael Walsh repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 235*83f4bc77SMichael Walsh # If we can't find ssh_pw, then we don't have our repo bin in PATH. 236b5839d00SMichael Walsh shell_rc, out_buf = gc.cmd_fnc_u("which ssh_pw", quiet=1, print_output=0, 237b5839d00SMichael Walsh show_err=0) 238b5839d00SMichael Walsh if shell_rc != 0: 239*83f4bc77SMichael Walsh os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 240*83f4bc77SMichael Walsh # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 241*83f4bc77SMichael Walsh if robot_pgm_dir_path not in sys.path: 242*83f4bc77SMichael Walsh sys.path.append(robot_pgm_dir_path) 243*83f4bc77SMichael Walsh PYTHONPATH = os.environ.get("PYTHONPATH", "") 244*83f4bc77SMichael Walsh if PYTHONPATH == "": 245*83f4bc77SMichael Walsh os.environ['PYTHONPATH'] = robot_pgm_dir_path 246*83f4bc77SMichael Walsh else: 247*83f4bc77SMichael Walsh os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 2486741f740SMichael Walsh 2496741f740SMichael Walsh validate_parms() 2506741f740SMichael Walsh 2516741f740SMichael Walsh grp.rqprint_pgm_header() 2526741f740SMichael Walsh 253b5839d00SMichael Walsh grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE") 25411cfc8c0SMichael Walsh 25585678948SMichael Walsh initial_plug_in_setup() 25685678948SMichael Walsh 2576741f740SMichael Walsh plug_in_setup() 2586741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 2596741f740SMichael Walsh call_point='setup') 2606741f740SMichael Walsh if rc != 0: 2616741f740SMichael Walsh error_message = "Plug-in setup failed.\n" 2626741f740SMichael Walsh grp.rprint_error_report(error_message) 2636741f740SMichael Walsh BuiltIn().fail(error_message) 2646741f740SMichael Walsh # Setting cp_setup_called lets our Teardown know that it needs to call 2656741f740SMichael Walsh # the cleanup plug-in call point. 2666741f740SMichael Walsh cp_setup_called = 1 2676741f740SMichael Walsh 2686741f740SMichael Walsh # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 2696741f740SMichael Walsh BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 27085678948SMichael Walsh # FFDC_LOG_PATH is used by "FFDC" keyword. 27185678948SMichael Walsh BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 2726741f740SMichael Walsh 273b5839d00SMichael Walsh gp.dprint_var(boot_table, 1) 274b5839d00SMichael Walsh gp.dprint_var(boot_lists) 2750bbd860fSMichael Walsh 2760bbd860fSMichael Walsh############################################################################### 2770bbd860fSMichael Walsh 2780bbd860fSMichael Walsh 2790bbd860fSMichael Walsh############################################################################### 2806741f740SMichael Walshdef validate_parms(): 2810bbd860fSMichael Walsh 2820bbd860fSMichael Walsh r""" 2836741f740SMichael Walsh Validate all program parameters. 2840bbd860fSMichael Walsh """ 2850bbd860fSMichael Walsh 286b5839d00SMichael Walsh process_pgm_parms() 2870bbd860fSMichael Walsh 288b5839d00SMichael Walsh gp.qprintn() 289b5839d00SMichael Walsh 290b5839d00SMichael Walsh global openbmc_model 2916741f740SMichael Walsh grv.rvalid_value("openbmc_host") 2926741f740SMichael Walsh grv.rvalid_value("openbmc_username") 2936741f740SMichael Walsh grv.rvalid_value("openbmc_password") 2946741f740SMichael Walsh if os_host != "": 2956741f740SMichael Walsh grv.rvalid_value("os_username") 2966741f740SMichael Walsh grv.rvalid_value("os_password") 2970bbd860fSMichael Walsh 2986741f740SMichael Walsh if pdu_host != "": 2996741f740SMichael Walsh grv.rvalid_value("pdu_username") 3006741f740SMichael Walsh grv.rvalid_value("pdu_password") 3016741f740SMichael Walsh grv.rvalid_integer("pdu_slot_no") 3026741f740SMichael Walsh if openbmc_serial_host != "": 3036741f740SMichael Walsh grv.rvalid_integer("openbmc_serial_port") 304b5839d00SMichael Walsh if openbmc_model == "": 305b5839d00SMichael Walsh status, ret_values =\ 306b5839d00SMichael Walsh grk.run_key_u("Get BMC System Model") 307b5839d00SMichael Walsh openbmc_model = ret_values 308b5839d00SMichael Walsh BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 3096741f740SMichael Walsh grv.rvalid_value("openbmc_model") 310b5839d00SMichael Walsh grv.rvalid_integer("max_num_tests") 3116741f740SMichael Walsh grv.rvalid_integer("boot_pass") 3126741f740SMichael Walsh grv.rvalid_integer("boot_fail") 3136741f740SMichael Walsh 3146741f740SMichael Walsh plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 3156741f740SMichael Walsh BuiltIn().set_global_variable("${plug_in_packages_list}", 3166741f740SMichael Walsh plug_in_packages_list) 3176741f740SMichael Walsh 318b5839d00SMichael Walsh grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 319a20da401SMichael Walsh if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 3206741f740SMichael Walsh error_message = "You must provide either a value for either the" +\ 3216741f740SMichael Walsh " boot_list or the boot_stack parm.\n" 3226741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 3236741f740SMichael Walsh 3246741f740SMichael Walsh valid_boot_list(boot_list, valid_boot_types) 3256741f740SMichael Walsh valid_boot_list(boot_stack, valid_boot_types) 3266741f740SMichael Walsh 32711cfc8c0SMichael Walsh selected_PDU_boots = list(set(boot_list + boot_stack) & 32811cfc8c0SMichael Walsh set(boot_lists['PDU_reboot'])) 32911cfc8c0SMichael Walsh 33011cfc8c0SMichael Walsh if len(selected_PDU_boots) > 0 and pdu_host == "": 33111cfc8c0SMichael Walsh error_message = "You have selected the following boots which" +\ 33211cfc8c0SMichael Walsh " require a PDU host but no value for pdu_host:\n" 33311cfc8c0SMichael Walsh error_message += gp.sprint_var(selected_PDU_boots) 33411cfc8c0SMichael Walsh error_message += gp.sprint_var(pdu_host, 2) 33511cfc8c0SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 33611cfc8c0SMichael Walsh 3376741f740SMichael Walsh return 3380bbd860fSMichael Walsh 3390bbd860fSMichael Walsh############################################################################### 3400bbd860fSMichael Walsh 3410bbd860fSMichael Walsh 3420bbd860fSMichael Walsh############################################################################### 3436741f740SMichael Walshdef my_get_state(): 3440bbd860fSMichael Walsh 3450bbd860fSMichael Walsh r""" 3466741f740SMichael Walsh Get the system state plus a little bit of wrapping. 3470bbd860fSMichael Walsh """ 3480bbd860fSMichael Walsh 3496741f740SMichael Walsh global state 3506741f740SMichael Walsh 3516741f740SMichael Walsh req_states = ['epoch_seconds'] + st.default_req_states 3526741f740SMichael Walsh 353b5839d00SMichael Walsh gp.qprint_timen("Getting system state.") 3546741f740SMichael Walsh if test_mode: 3556741f740SMichael Walsh state['epoch_seconds'] = int(time.time()) 3566741f740SMichael Walsh else: 357b5839d00SMichael Walsh state = st.get_state(req_states=req_states, quiet=quiet) 358b5839d00SMichael Walsh gp.qprint_var(state) 359341c21ebSMichael Walsh 360341c21ebSMichael Walsh############################################################################### 361341c21ebSMichael Walsh 362341c21ebSMichael Walsh 363341c21ebSMichael Walsh############################################################################### 3646741f740SMichael Walshdef select_boot(): 365341c21ebSMichael Walsh 366341c21ebSMichael Walsh r""" 367341c21ebSMichael Walsh Select a boot test to be run based on our current state and return the 368341c21ebSMichael Walsh chosen boot type. 369341c21ebSMichael Walsh 370341c21ebSMichael Walsh Description of arguments: 3716741f740SMichael Walsh state The state of the machine. 372341c21ebSMichael Walsh """ 373341c21ebSMichael Walsh 37430dadae2SMichael Walsh global boot_stack 37530dadae2SMichael Walsh 376b5839d00SMichael Walsh gp.qprint_timen("Selecting a boot test.") 3776741f740SMichael Walsh 3786741f740SMichael Walsh my_get_state() 3796741f740SMichael Walsh 3806741f740SMichael Walsh stack_popped = 0 3816741f740SMichael Walsh if len(boot_stack) > 0: 3826741f740SMichael Walsh stack_popped = 1 383b5839d00SMichael Walsh gp.qprint_dashes() 384b5839d00SMichael Walsh gp.qprint_var(boot_stack) 385b5839d00SMichael Walsh gp.qprint_dashes() 386b5839d00SMichael Walsh skip_boot_printed = 0 387b5839d00SMichael Walsh while len(boot_stack) > 0: 3886741f740SMichael Walsh boot_candidate = boot_stack.pop() 389b5839d00SMichael Walsh if stack_mode == 'normal': 390b5839d00SMichael Walsh break 391b5839d00SMichael Walsh else: 392b5839d00SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['end']): 393b5839d00SMichael Walsh if not skip_boot_printed: 394b5839d00SMichael Walsh gp.print_var(stack_mode) 395b5839d00SMichael Walsh gp.printn() 396b5839d00SMichael Walsh gp.print_timen("Skipping the following boot tests" + 397b5839d00SMichael Walsh " which are unnecessary since their" + 398b5839d00SMichael Walsh " required end states match the" + 399b5839d00SMichael Walsh " current machine state:") 400b5839d00SMichael Walsh skip_boot_printed = 1 401b5839d00SMichael Walsh gp.print_var(boot_candidate) 402b5839d00SMichael Walsh boot_candidate = "" 403b5839d00SMichael Walsh if boot_candidate == "": 404b5839d00SMichael Walsh gp.qprint_dashes() 405b5839d00SMichael Walsh gp.qprint_var(boot_stack) 406b5839d00SMichael Walsh gp.qprint_dashes() 407b5839d00SMichael Walsh return boot_candidate 4086741f740SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['start']): 409b5839d00SMichael Walsh gp.qprint_timen("The machine state is valid for a '" + 4106741f740SMichael Walsh boot_candidate + "' boot test.") 411b5839d00SMichael Walsh gp.qprint_dashes() 412b5839d00SMichael Walsh gp.qprint_var(boot_stack) 413b5839d00SMichael Walsh gp.qprint_dashes() 4146741f740SMichael Walsh return boot_candidate 415341c21ebSMichael Walsh else: 416b5839d00SMichael Walsh gp.qprint_timen("The machine state does not match the required" + 417b5839d00SMichael Walsh " starting state for a '" + boot_candidate + 418b5839d00SMichael Walsh "' boot test:") 419b5839d00SMichael Walsh gp.print_varx("boot_table[" + boot_candidate + "][start]", 420b5839d00SMichael Walsh boot_table[boot_candidate]['start'], 1) 4216741f740SMichael Walsh boot_stack.append(boot_candidate) 4226741f740SMichael Walsh popped_boot = boot_candidate 4236741f740SMichael Walsh 4246741f740SMichael Walsh # Loop through your list selecting a boot_candidates 4256741f740SMichael Walsh boot_candidates = [] 4266741f740SMichael Walsh for boot_candidate in boot_list: 4276741f740SMichael Walsh if st.compare_states(state, boot_table[boot_candidate]['start']): 4286741f740SMichael Walsh if stack_popped: 4296741f740SMichael Walsh if st.compare_states(boot_table[boot_candidate]['end'], 4306741f740SMichael Walsh boot_table[popped_boot]['start']): 4316741f740SMichael Walsh boot_candidates.append(boot_candidate) 432341c21ebSMichael Walsh else: 4336741f740SMichael Walsh boot_candidates.append(boot_candidate) 4346741f740SMichael Walsh 4356741f740SMichael Walsh if len(boot_candidates) == 0: 436b5839d00SMichael Walsh gp.qprint_timen("The user's boot list contained no boot tests" + 4376741f740SMichael Walsh " which are valid for the current machine state.") 4386741f740SMichael Walsh boot_candidate = default_power_on 4396741f740SMichael Walsh if not st.compare_states(state, boot_table[default_power_on]['start']): 4406741f740SMichael Walsh boot_candidate = default_power_off 4416741f740SMichael Walsh boot_candidates.append(boot_candidate) 442b5839d00SMichael Walsh gp.qprint_timen("Using default '" + boot_candidate + 443b5839d00SMichael Walsh "' boot type to transition to valid state.") 4446741f740SMichael Walsh 445b5839d00SMichael Walsh gp.dprint_var(boot_candidates) 4466741f740SMichael Walsh 4476741f740SMichael Walsh # Randomly select a boot from the candidate list. 4486741f740SMichael Walsh boot = random.choice(boot_candidates) 449341c21ebSMichael Walsh 450341c21ebSMichael Walsh return boot 4510bbd860fSMichael Walsh 4520bbd860fSMichael Walsh############################################################################### 45355302295SMichael Walsh 45455302295SMichael Walsh 45555302295SMichael Walsh############################################################################### 456341c21ebSMichael Walshdef print_last_boots(): 457341c21ebSMichael Walsh 458341c21ebSMichael Walsh r""" 459341c21ebSMichael Walsh Print the last ten boots done with their time stamps. 460341c21ebSMichael Walsh """ 461341c21ebSMichael Walsh 462341c21ebSMichael Walsh # indent 0, 90 chars wide, linefeed, char is "=" 463b5839d00SMichael Walsh gp.qprint_dashes(0, 90) 464b5839d00SMichael Walsh gp.qprintn("Last 10 boots:\n") 465341c21ebSMichael Walsh 466341c21ebSMichael Walsh for boot_entry in last_ten: 467341c21ebSMichael Walsh grp.rqprint(boot_entry) 468b5839d00SMichael Walsh gp.qprint_dashes(0, 90) 469341c21ebSMichael Walsh 470341c21ebSMichael Walsh############################################################################### 471341c21ebSMichael Walsh 472341c21ebSMichael Walsh 473341c21ebSMichael Walsh############################################################################### 474341c21ebSMichael Walshdef print_defect_report(): 475341c21ebSMichael Walsh 476341c21ebSMichael Walsh r""" 477341c21ebSMichael Walsh Print a defect report. 478341c21ebSMichael Walsh """ 479341c21ebSMichael Walsh 480b5839d00SMichael Walsh gp.qprintn() 481341c21ebSMichael Walsh # indent=0, width=90, linefeed=1, char="=" 482b5839d00SMichael Walsh gp.qprint_dashes(0, 90, 1, "=") 483b5839d00SMichael Walsh gp.qprintn("Copy this data to the defect:\n") 484341c21ebSMichael Walsh 485341c21ebSMichael Walsh grp.rqpvars(*parm_list) 486341c21ebSMichael Walsh 487b5839d00SMichael Walsh gp.qprintn() 488341c21ebSMichael Walsh 489341c21ebSMichael Walsh print_last_boots() 490b5839d00SMichael Walsh gp.qprintn() 491b5839d00SMichael Walsh gp.qprint_var(state) 492341c21ebSMichael Walsh 493341c21ebSMichael Walsh # At some point I'd like to have the 'Call FFDC Methods' return a list 494341c21ebSMichael Walsh # of files it has collected. In that case, the following "ls" command 495341c21ebSMichael Walsh # would no longer be needed. For now, however, glob shows the files 496341c21ebSMichael Walsh # named in FFDC_LIST_FILE_PATH so I will refrain from printing those 497341c21ebSMichael Walsh # out (so we don't see duplicates in the list). 498341c21ebSMichael Walsh 499341c21ebSMichael Walsh LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}") 500341c21ebSMichael Walsh 501341c21ebSMichael Walsh output = '\n'.join(glob.glob(LOG_PREFIX + '*')) 502341c21ebSMichael Walsh try: 5036741f740SMichael Walsh ffdc_list = open(ffdc_list_file_path, 'r') 504341c21ebSMichael Walsh except IOError: 505341c21ebSMichael Walsh ffdc_list = "" 506341c21ebSMichael Walsh 507b5839d00SMichael Walsh gp.qprintn() 508b5839d00SMichael Walsh gp.qprintn("FFDC data files:") 509341c21ebSMichael Walsh if status_file_path != "": 510b5839d00SMichael Walsh gp.qprintn(status_file_path) 511341c21ebSMichael Walsh 512b5839d00SMichael Walsh gp.qprintn(output) 513b5839d00SMichael Walsh # gp.qprintn(ffdc_list) 514b5839d00SMichael Walsh gp.qprintn() 515341c21ebSMichael Walsh 516b5839d00SMichael Walsh gp.qprint_dashes(0, 90, 1, "=") 517341c21ebSMichael Walsh 518341c21ebSMichael Walsh############################################################################### 5196741f740SMichael Walsh 5206741f740SMichael Walsh 5216741f740SMichael Walsh############################################################################### 5226741f740SMichael Walshdef my_ffdc(): 5236741f740SMichael Walsh 5246741f740SMichael Walsh r""" 5256741f740SMichael Walsh Collect FFDC data. 5266741f740SMichael Walsh """ 5276741f740SMichael Walsh 5286741f740SMichael Walsh global state 5296741f740SMichael Walsh 5306741f740SMichael Walsh plug_in_setup() 5316741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 5326741f740SMichael Walsh call_point='ffdc', stop_on_plug_in_failure=1) 5336741f740SMichael Walsh 5346741f740SMichael Walsh AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 535*83f4bc77SMichael Walsh status, ret_values = grk.run_key_u("FFDC ffdc_prefix=" + 536*83f4bc77SMichael Walsh AUTOBOOT_FFDC_PREFIX + 537*83f4bc77SMichael Walsh " ffdc_function_list=" + 538*83f4bc77SMichael Walsh ffdc_function_list, ignore=1) 539*83f4bc77SMichael Walsh if status != 'PASS': 5403328caffSMichael Walsh gp.print_error("Call to ffdc failed.\n") 5416741f740SMichael Walsh 5426741f740SMichael Walsh my_get_state() 5436741f740SMichael Walsh 5446741f740SMichael Walsh print_defect_report() 5456741f740SMichael Walsh 5466741f740SMichael Walsh############################################################################### 5476741f740SMichael Walsh 5486741f740SMichael Walsh 5496741f740SMichael Walsh############################################################################### 5506741f740SMichael Walshdef print_test_start_message(boot_keyword): 5516741f740SMichael Walsh 5526741f740SMichael Walsh r""" 5536741f740SMichael Walsh Print a message indicating what boot test is about to run. 5546741f740SMichael Walsh 5556741f740SMichael Walsh Description of arguments: 5566741f740SMichael Walsh boot_keyword The name of the boot which is to be run 5576741f740SMichael Walsh (e.g. "BMC Power On"). 5586741f740SMichael Walsh """ 5596741f740SMichael Walsh 5606741f740SMichael Walsh global last_ten 5616741f740SMichael Walsh 5626741f740SMichael Walsh doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 563b5839d00SMichael Walsh gp.qprint(doing_msg) 5646741f740SMichael Walsh 5656741f740SMichael Walsh last_ten.append(doing_msg) 5666741f740SMichael Walsh 5676741f740SMichael Walsh if len(last_ten) > 10: 5686741f740SMichael Walsh del last_ten[0] 5696741f740SMichael Walsh 5706741f740SMichael Walsh############################################################################### 5716741f740SMichael Walsh 5726741f740SMichael Walsh 5736741f740SMichael Walsh############################################################################### 5746741f740SMichael Walshdef run_boot(boot): 5756741f740SMichael Walsh 5766741f740SMichael Walsh r""" 5776741f740SMichael Walsh Run the specified boot. 5786741f740SMichael Walsh 5796741f740SMichael Walsh Description of arguments: 5806741f740SMichael Walsh boot The name of the boot test to be performed. 5816741f740SMichael Walsh """ 5826741f740SMichael Walsh 5836741f740SMichael Walsh global state 5846741f740SMichael Walsh 5856741f740SMichael Walsh print_test_start_message(boot) 5866741f740SMichael Walsh 5876741f740SMichael Walsh plug_in_setup() 5886741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 5896741f740SMichael Walsh grpi.rprocess_plug_in_packages(call_point="pre_boot") 5906741f740SMichael Walsh if rc != 0: 5916741f740SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" +\ 5926741f740SMichael Walsh gp.sprint_var(rc, 1) 5936741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 5946741f740SMichael Walsh 5956741f740SMichael Walsh if test_mode: 5966741f740SMichael Walsh # In test mode, we'll pretend the boot worked by assigning its 5976741f740SMichael Walsh # required end state to the default state value. 59830dadae2SMichael Walsh state = st.strip_anchor_state(boot_table[boot]['end']) 5996741f740SMichael Walsh else: 6006741f740SMichael Walsh # Assertion: We trust that the state data was made fresh by the 6016741f740SMichael Walsh # caller. 6026741f740SMichael Walsh 603b5839d00SMichael Walsh gp.qprintn() 6046741f740SMichael Walsh 6056741f740SMichael Walsh if boot_table[boot]['method_type'] == "keyword": 6060b93fbf8SMichael Walsh rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 607b5839d00SMichael Walsh boot_table[boot]['method'], 608b5839d00SMichael Walsh quiet=quiet) 6096741f740SMichael Walsh 6106741f740SMichael Walsh if boot_table[boot]['bmc_reboot']: 6116741f740SMichael Walsh st.wait_for_comm_cycle(int(state['epoch_seconds'])) 61230dadae2SMichael Walsh plug_in_setup() 61330dadae2SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 61430dadae2SMichael Walsh grpi.rprocess_plug_in_packages(call_point="post_reboot") 61530dadae2SMichael Walsh if rc != 0: 6160b93fbf8SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" 6170b93fbf8SMichael Walsh error_message += gp.sprint_var(rc, 1) 61830dadae2SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 6196741f740SMichael Walsh else: 6206741f740SMichael Walsh match_state = st.anchor_state(state) 6216741f740SMichael Walsh del match_state['epoch_seconds'] 6226741f740SMichael Walsh # Wait for the state to change in any way. 6236741f740SMichael Walsh st.wait_state(match_state, wait_time=state_change_timeout, 6246741f740SMichael Walsh interval="3 seconds", invert=1) 6256741f740SMichael Walsh 626b5839d00SMichael Walsh gp.qprintn() 6276741f740SMichael Walsh if boot_table[boot]['end']['chassis'] == "Off": 6286741f740SMichael Walsh boot_timeout = power_off_timeout 6296741f740SMichael Walsh else: 6306741f740SMichael Walsh boot_timeout = power_on_timeout 6316741f740SMichael Walsh st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 6326741f740SMichael Walsh interval="3 seconds") 6336741f740SMichael Walsh 6346741f740SMichael Walsh plug_in_setup() 6356741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = \ 6366741f740SMichael Walsh grpi.rprocess_plug_in_packages(call_point="post_boot") 6376741f740SMichael Walsh if rc != 0: 6386741f740SMichael Walsh error_message = "Plug-in failed with non-zero return code.\n" +\ 6396741f740SMichael Walsh gp.sprint_var(rc, 1) 6406741f740SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 6416741f740SMichael Walsh 6426741f740SMichael Walsh############################################################################### 6436741f740SMichael Walsh 6446741f740SMichael Walsh 6456741f740SMichael Walsh############################################################################### 6466741f740SMichael Walshdef test_loop_body(): 6476741f740SMichael Walsh 6486741f740SMichael Walsh r""" 6496741f740SMichael Walsh The main loop body for the loop in main_py. 6506741f740SMichael Walsh 6516741f740SMichael Walsh Description of arguments: 6526741f740SMichael Walsh boot_count The iteration number (starts at 1). 6536741f740SMichael Walsh """ 6546741f740SMichael Walsh 6556741f740SMichael Walsh global boot_count 6566741f740SMichael Walsh global state 6576741f740SMichael Walsh global next_boot 6586741f740SMichael Walsh global boot_success 6596741f740SMichael Walsh 660b5839d00SMichael Walsh gp.qprintn() 6616741f740SMichael Walsh 6626741f740SMichael Walsh next_boot = select_boot() 663b5839d00SMichael Walsh if next_boot == "": 664b5839d00SMichael Walsh return True 6656741f740SMichael Walsh 666b5839d00SMichael Walsh boot_count += 1 667b5839d00SMichael Walsh gp.qprint_timen("Starting boot " + str(boot_count) + ".") 6686741f740SMichael Walsh 6696741f740SMichael Walsh # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 6706741f740SMichael Walsh try: 6716741f740SMichael Walsh os.remove(ffdc_list_file_path) 6726741f740SMichael Walsh except OSError: 6736741f740SMichael Walsh pass 6746741f740SMichael Walsh 6756741f740SMichael Walsh cmd_buf = ["run_boot", next_boot] 6766741f740SMichael Walsh boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 6776741f740SMichael Walsh if boot_status == "FAIL": 678b5839d00SMichael Walsh gp.qprint(msg) 6796741f740SMichael Walsh 680b5839d00SMichael Walsh gp.qprintn() 6816741f740SMichael Walsh if boot_status == "PASS": 6826741f740SMichael Walsh boot_success = 1 683b5839d00SMichael Walsh gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.") 6846741f740SMichael Walsh else: 6856741f740SMichael Walsh boot_success = 0 686b5839d00SMichael Walsh gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.") 6876741f740SMichael Walsh 6886741f740SMichael Walsh boot_results.update(next_boot, boot_status) 6896741f740SMichael Walsh 6906741f740SMichael Walsh plug_in_setup() 6916741f740SMichael Walsh # NOTE: A post_test_case call point failure is NOT counted as a boot 6926741f740SMichael Walsh # failure. 6936741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 6946741f740SMichael Walsh call_point='post_test_case', stop_on_plug_in_failure=1) 6956741f740SMichael Walsh 6966741f740SMichael Walsh plug_in_setup() 6976741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 6986741f740SMichael Walsh call_point='ffdc_check', shell_rc=0x00000200, 6996741f740SMichael Walsh stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 7006741f740SMichael Walsh if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 701*83f4bc77SMichael Walsh status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 702*83f4bc77SMichael Walsh if status != 'PASS': 7033328caffSMichael Walsh gp.print_error("Call to my_ffdc failed.\n") 7046741f740SMichael Walsh 705d139f286SMichael Walsh # We need to purge error logs between boots or they build up. 706b5839d00SMichael Walsh grk.run_key("Delete Error logs", ignore=1) 707d139f286SMichael Walsh 708952f9b09SMichael Walsh boot_results.print_report() 709b5839d00SMichael Walsh gp.qprint_timen("Finished boot " + str(boot_count) + ".") 710952f9b09SMichael Walsh 7116741f740SMichael Walsh plug_in_setup() 7126741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 7136741f740SMichael Walsh call_point='stop_check') 7146741f740SMichael Walsh if rc != 0: 7156741f740SMichael Walsh error_message = "Stopping as requested by user.\n" 7166741f740SMichael Walsh grp.rprint_error_report(error_message) 7176741f740SMichael Walsh BuiltIn().fail(error_message) 7186741f740SMichael Walsh 719d139f286SMichael Walsh # This should help prevent ConnectionErrors. 720b5839d00SMichael Walsh grk.run_key_u("Delete All Sessions") 721d139f286SMichael Walsh 7226741f740SMichael Walsh return True 7236741f740SMichael Walsh 7246741f740SMichael Walsh############################################################################### 7256741f740SMichael Walsh 7266741f740SMichael Walsh 7276741f740SMichael Walsh############################################################################### 728*83f4bc77SMichael Walshdef obmc_boot_test_teardown(): 7296741f740SMichael Walsh 7306741f740SMichael Walsh r""" 731c9116811SMichael Walsh Clean up after the Main keyword. 7326741f740SMichael Walsh """ 7336741f740SMichael Walsh 7346741f740SMichael Walsh if cp_setup_called: 7356741f740SMichael Walsh plug_in_setup() 7366741f740SMichael Walsh rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 7376741f740SMichael Walsh call_point='cleanup', stop_on_plug_in_failure=1) 7386741f740SMichael Walsh 7390b93fbf8SMichael Walsh # Save boot_results object to a file in case it is needed again. 740b5839d00SMichael Walsh gp.qprint_timen("Saving boot_results to the following path.") 741b5839d00SMichael Walsh gp.qprint_var(boot_results_file_path) 7420b93fbf8SMichael Walsh pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 7430b93fbf8SMichael Walsh pickle.HIGHEST_PROTOCOL) 7440b93fbf8SMichael Walsh 7456741f740SMichael Walsh############################################################################### 7466741f740SMichael Walsh 7476741f740SMichael Walsh 7486741f740SMichael Walsh############################################################################### 749c9116811SMichael Walshdef test_teardown(): 750c9116811SMichael Walsh 751c9116811SMichael Walsh r""" 752c9116811SMichael Walsh Clean up after this test case. 753c9116811SMichael Walsh """ 754c9116811SMichael Walsh 755c9116811SMichael Walsh gp.qprintn() 756c9116811SMichael Walsh cmd_buf = ["Print Error", 757c9116811SMichael Walsh "A keyword timeout occurred ending this program.\n"] 758c9116811SMichael Walsh BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 759c9116811SMichael Walsh 760b5839d00SMichael Walsh grp.rqprint_pgm_footer() 761b5839d00SMichael Walsh 762c9116811SMichael Walsh############################################################################### 763c9116811SMichael Walsh 764c9116811SMichael Walsh 765c9116811SMichael Walsh############################################################################### 766*83f4bc77SMichael Walshdef obmc_boot_test_py(alt_boot_stack=None): 7676741f740SMichael Walsh 7686741f740SMichael Walsh r""" 7696741f740SMichael Walsh Do main program processing. 7706741f740SMichael Walsh """ 7716741f740SMichael Walsh 772b5839d00SMichael Walsh if alt_boot_stack is not None: 773b5839d00SMichael Walsh BuiltIn().set_global_variable("${boot_stack}", alt_boot_stack) 774b5839d00SMichael Walsh 7756741f740SMichael Walsh setup() 7766741f740SMichael Walsh 777a20da401SMichael Walsh if ffdc_only: 778a20da401SMichael Walsh gp.qprint_timen("Caller requested ffdc_only.") 779*83f4bc77SMichael Walsh grk.run_key_u("my_ffdc") 780a20da401SMichael Walsh 7816741f740SMichael Walsh # Process caller's boot_stack. 7826741f740SMichael Walsh while (len(boot_stack) > 0): 7836741f740SMichael Walsh test_loop_body() 7846741f740SMichael Walsh 785b5839d00SMichael Walsh gp.qprint_timen("Finished processing stack.") 78630dadae2SMichael Walsh 7876741f740SMichael Walsh # Process caller's boot_list. 7886741f740SMichael Walsh if len(boot_list) > 0: 7896741f740SMichael Walsh for ix in range(1, max_num_tests + 1): 7906741f740SMichael Walsh test_loop_body() 7916741f740SMichael Walsh 792b5839d00SMichael Walsh gp.qprint_timen("Completed all requested boot tests.") 793b5839d00SMichael Walsh 794b5839d00SMichael Walsh boot_pass, boot_fail = boot_results.return_total_pass_fail() 795b5839d00SMichael Walsh if boot_fail > boot_fail_threshold: 796b5839d00SMichael Walsh error_message = "Boot failures exceed the boot failure" +\ 797b5839d00SMichael Walsh " threshold:\n" +\ 798b5839d00SMichael Walsh gp.sprint_var(boot_fail) +\ 799b5839d00SMichael Walsh gp.sprint_var(boot_fail_threshold) 800b5839d00SMichael Walsh BuiltIn().fail(gp.sprint_error(error_message)) 8016741f740SMichael Walsh 8026741f740SMichael Walsh############################################################################### 803