1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34 35base_path = os.path.dirname(os.path.dirname( 36 imp.find_module("gen_robot_print")[1])) +\ 37 os.sep 38sys.path.append(base_path + "extended/") 39import run_keyword as rk 40 41# Setting master_pid correctly influences the behavior of plug-ins like 42# DB_Logging 43program_pid = os.getpid() 44master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 45pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 46 47# Set up boot data structures. 48os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 49 50boot_lists = read_boot_lists() 51 52# The maximum number of entries that can be in the boot_history global variable. 53max_boot_history = 10 54boot_history = [] 55 56state = st.return_state_constant('default_state') 57cp_setup_called = 0 58next_boot = "" 59base_tool_dir_path = os.path.normpath(os.environ.get( 60 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 61 62ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 63boot_success = 0 64status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 65if status_dir_path != "": 66 status_dir_path = os.path.normpath(status_dir_path) + os.sep 67redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 68redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 69if redfish_supported: 70 redfish = BuiltIn().get_library_instance('redfish') 71 default_power_on = "Redfish Power On" 72 default_power_off = "Redfish Power Off" 73 if redfish_rest_supported: 74 delete_errlogs_cmd = "Delete Error Logs" 75 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 76 else: 77 delete_errlogs_cmd = "Redfish Purge Event Log" 78 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 79else: 80 default_power_on = "REST Power On" 81 default_power_off = "REST Power Off" 82 delete_errlogs_cmd = "Delete Error Logs" 83 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 84boot_count = 0 85 86LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 87AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 88ffdc_prefix = AUTOBOOT_FFDC_PREFIX 89boot_start_time = "" 90boot_end_time = "" 91save_stack = vs.var_stack('save_stack') 92main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 93 94 95def dump_ffdc_rc(): 96 r""" 97 Return the constant dump ffdc test return code value. 98 99 When a plug-in call point program returns this value, it indicates that 100 this program should collect FFDC. 101 """ 102 103 return 0x00000200 104 105 106def stop_test_rc(): 107 r""" 108 Return the constant stop test return code value. 109 110 When a plug-in call point program returns this value, it indicates that 111 this program should stop running. 112 """ 113 114 return 0x00000200 115 116 117def process_host(host, 118 host_var_name=""): 119 r""" 120 Process a host by getting the associated host name and IP address and 121 setting them in global variables. 122 123 If the caller does not pass the host_var_name, this function will try to 124 figure out the name of the variable used by the caller for the host parm. 125 Callers are advised to explicitly specify the host_var_name when calling 126 with an exec command. In such cases, the get_arg_name cannot figure out 127 the host variable name. 128 129 This function will then create similar global variable names by 130 removing "_host" and appending "_host_name" or "_ip" to the host variable 131 name. 132 133 Example: 134 135 If a call is made like this: 136 process_host(openbmc_host) 137 138 Global variables openbmc_host_name and openbmc_ip will be set. 139 140 Description of argument(s): 141 host A host name or IP. The name of the variable used should 142 have a suffix of "_host". 143 host_var_name The name of the variable being used as the host parm. 144 """ 145 146 if host_var_name == "": 147 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 148 149 host_name_var_name = re.sub("host", "host_name", host_var_name) 150 ip_var_name = re.sub("host", "ip", host_var_name) 151 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 152 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 153 host + "')" 154 exec(cmd_buf) 155 156 157def process_pgm_parms(): 158 r""" 159 Process the program parameters by assigning them all to corresponding 160 globals. Also, set some global values that depend on program parameters. 161 """ 162 163 # Program parameter processing. 164 # Assign all program parms to python variables which are global to this 165 # module. 166 167 global parm_list 168 parm_list = BuiltIn().get_variable_value("${parm_list}") 169 # The following subset of parms should be processed as integers. 170 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 171 'boot_fail_threshold', 'delete_errlogs', 172 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 173 'test_mode', 'debug'] 174 for parm in parm_list: 175 if parm in int_list: 176 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 177 "}\", \"0\"))" 178 else: 179 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 180 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 181 gp.dpissuing(cmd_buf) 182 exec(cmd_buf) 183 if re.match(r".*_host$", parm): 184 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 185 exec(cmd_buf) 186 if re.match(r".*_password$", parm): 187 # Register the value of any parm whose name ends in _password. 188 # This will cause the print functions to replace passwords with 189 # asterisks in the output. 190 cmd_buf = "gp.register_passwords(" + parm + ")" 191 exec(cmd_buf) 192 193 global ffdc_dir_path_style 194 global boot_list 195 global boot_stack 196 global boot_results_file_path 197 global boot_results 198 global boot_history 199 global ffdc_list_file_path 200 global ffdc_report_list_path 201 global ffdc_summary_list_path 202 global boot_table 203 global valid_boot_types 204 205 if ffdc_dir_path_style == "": 206 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 207 208 # Convert these program parms to lists for easier processing.. 209 boot_list = list(filter(None, boot_list.split(":"))) 210 boot_stack = list(filter(None, boot_stack.split(":"))) 211 212 boot_table = create_boot_table(boot_table_path, os_host=os_host) 213 valid_boot_types = create_valid_boot_list(boot_table) 214 215 cleanup_boot_results_file() 216 boot_results_file_path = create_boot_results_file_path(pgm_name, 217 openbmc_nickname, 218 master_pid) 219 220 if os.path.isfile(boot_results_file_path): 221 # We've been called before in this run so we'll load the saved 222 # boot_results and boot_history objects. 223 boot_results, boot_history =\ 224 pickle.load(open(boot_results_file_path, 'rb')) 225 else: 226 boot_results = boot_results(boot_table, boot_pass, boot_fail) 227 228 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 229 "/FFDC_FILE_LIST" 230 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 231 "/FFDC_REPORT_FILE_LIST" 232 233 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 234 "/FFDC_SUMMARY_FILE_LIST" 235 236 237def initial_plug_in_setup(): 238 r""" 239 Initialize all plug-in environment variables which do not change for the 240 duration of the program. 241 242 """ 243 244 global LOG_LEVEL 245 BuiltIn().set_log_level("NONE") 246 247 BuiltIn().set_global_variable("${master_pid}", master_pid) 248 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 249 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 250 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 251 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 252 ffdc_list_file_path) 253 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 254 ffdc_report_list_path) 255 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 256 ffdc_summary_list_path) 257 258 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 259 ffdc_dir_path_style) 260 BuiltIn().set_global_variable("${FFDC_CHECK}", 261 ffdc_check) 262 263 # For each program parameter, set the corresponding AUTOBOOT_ environment 264 # variable value. Also, set an AUTOBOOT_ environment variable for every 265 # element in additional_values. 266 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 267 "status_dir_path", "base_tool_dir_path", 268 "ffdc_list_file_path", "ffdc_report_list_path", 269 "ffdc_summary_list_path", "execdir", "redfish_supported"] 270 271 plug_in_vars = parm_list + additional_values 272 273 for var_name in plug_in_vars: 274 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 275 var_name = var_name.upper() 276 if var_value is None: 277 var_value = "" 278 os.environ["AUTOBOOT_" + var_name] = str(var_value) 279 280 BuiltIn().set_log_level(LOG_LEVEL) 281 282 # Make sure the ffdc list directory exists. 283 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 284 if not os.path.exists(ffdc_list_dir_path): 285 os.makedirs(ffdc_list_dir_path) 286 287 288def plug_in_setup(): 289 r""" 290 Initialize all changing plug-in environment variables for use by the 291 plug-in programs. 292 """ 293 294 global LOG_LEVEL 295 global test_really_running 296 297 BuiltIn().set_log_level("NONE") 298 299 boot_pass, boot_fail = boot_results.return_total_pass_fail() 300 if boot_pass > 1: 301 test_really_running = 1 302 else: 303 test_really_running = 0 304 305 BuiltIn().set_global_variable("${test_really_running}", 306 test_really_running) 307 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 308 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 309 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 310 BuiltIn().set_global_variable("${boot_success}", boot_success) 311 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 312 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 313 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 314 315 # For each program parameter, set the corresponding AUTOBOOT_ environment 316 # variable value. Also, set an AUTOBOOT_ environment variable for every 317 # element in additional_values. 318 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 319 "boot_fail", "test_really_running", "ffdc_prefix", 320 "boot_start_time", "boot_end_time"] 321 322 plug_in_vars = additional_values 323 324 for var_name in plug_in_vars: 325 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 326 var_name = var_name.upper() 327 if var_value is None: 328 var_value = "" 329 os.environ["AUTOBOOT_" + var_name] = str(var_value) 330 331 if debug: 332 shell_rc, out_buf = \ 333 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 334 335 BuiltIn().set_log_level(LOG_LEVEL) 336 337 338def pre_boot_plug_in_setup(): 339 340 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 341 try: 342 os.remove(ffdc_list_file_path) 343 except OSError: 344 pass 345 346 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 347 try: 348 os.remove(ffdc_report_list_path) 349 except OSError: 350 pass 351 352 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 353 try: 354 os.remove(ffdc_summary_list_path) 355 except OSError: 356 pass 357 358 global ffdc_prefix 359 360 seconds = time.time() 361 loc_time = time.localtime(seconds) 362 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 363 364 ffdc_prefix = openbmc_nickname + "." + time_string 365 366 367def default_sigusr1(signal_number=0, 368 frame=None): 369 r""" 370 Handle SIGUSR1 by doing nothing. 371 372 This function assists in debugging SIGUSR1 processing by printing messages 373 to stdout and to the log.html file. 374 375 Description of argument(s): 376 signal_number The signal number (should always be 10 for SIGUSR1). 377 frame The frame data. 378 """ 379 380 gp.qprintn() 381 gp.qprint_executing() 382 gp.lprint_executing() 383 384 385def set_default_siguser1(): 386 r""" 387 Set the default_sigusr1 function to be the SIGUSR1 handler. 388 """ 389 390 gp.qprintn() 391 gp.qprint_executing() 392 gp.lprint_executing() 393 signal.signal(signal.SIGUSR1, default_sigusr1) 394 395 396def setup(): 397 r""" 398 Do general program setup tasks. 399 """ 400 401 global cp_setup_called 402 global transitional_boot_selected 403 404 gp.qprintn() 405 406 if redfish_supported: 407 redfish.login() 408 409 set_default_siguser1() 410 transitional_boot_selected = False 411 412 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 413 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 414 # If we can't find process_plug_in_packages.py, ssh_pw or 415 # validate_plug_ins.py, then we don't have our repo bin in PATH. 416 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 417 + " ssh_pw validate_plug_ins.py", quiet=1, 418 print_output=0, show_err=0) 419 if shell_rc != 0: 420 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 421 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 422 if robot_pgm_dir_path not in sys.path: 423 sys.path.append(robot_pgm_dir_path) 424 PYTHONPATH = os.environ.get("PYTHONPATH", "") 425 if PYTHONPATH == "": 426 os.environ['PYTHONPATH'] = robot_pgm_dir_path 427 else: 428 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 429 430 validate_parms() 431 432 gp.qprint_pgm_header() 433 434 grk.run_key_u(default_set_power_policy) 435 436 initial_plug_in_setup() 437 438 plug_in_setup() 439 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 440 call_point='setup') 441 if rc != 0: 442 error_message = "Plug-in setup failed.\n" 443 gp.print_error_report(error_message) 444 BuiltIn().fail(error_message) 445 # Setting cp_setup_called lets our Teardown know that it needs to call 446 # the cleanup plug-in call point. 447 cp_setup_called = 1 448 449 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 450 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 451 # FFDC_LOG_PATH is used by "FFDC" keyword. 452 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 453 454 # Also printed by FFDC. 455 global host_name 456 global host_ip 457 host = socket.gethostname() 458 host_name, host_ip = gm.get_host_name_ip(host) 459 460 gp.dprint_var(boot_table) 461 gp.dprint_var(boot_lists) 462 463 464def validate_parms(): 465 r""" 466 Validate all program parameters. 467 """ 468 469 process_pgm_parms() 470 471 gp.qprintn() 472 473 global openbmc_model 474 if openbmc_model == "": 475 status, ret_values =\ 476 grk.run_key_u("Get BMC System Model") 477 openbmc_model = ret_values 478 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 479 gv.set_exit_on_error(True) 480 gv.valid_value(openbmc_host) 481 gv.valid_value(openbmc_username) 482 gv.valid_value(openbmc_password) 483 gv.valid_value(rest_username) 484 gv.valid_value(rest_password) 485 gv.valid_value(ipmi_username) 486 gv.valid_value(ipmi_password) 487 if os_host != "": 488 gv.valid_value(os_username) 489 gv.valid_value(os_password) 490 if pdu_host != "": 491 gv.valid_value(pdu_username) 492 gv.valid_value(pdu_password) 493 gv.valid_integer(pdu_slot_no) 494 if openbmc_serial_host != "": 495 gv.valid_integer(openbmc_serial_port) 496 gv.valid_value(openbmc_model) 497 gv.valid_integer(max_num_tests) 498 gv.valid_integer(boot_pass) 499 gv.valid_integer(boot_fail) 500 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 501 BuiltIn().set_global_variable("${plug_in_packages_list}", 502 plug_in_packages_list) 503 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 504 gv.set_exit_on_error(False) 505 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 506 error_message = "You must provide either a value for either the" +\ 507 " boot_list or the boot_stack parm.\n" 508 BuiltIn().fail(gp.sprint_error(error_message)) 509 valid_boot_list(boot_list, valid_boot_types) 510 valid_boot_list(boot_stack, valid_boot_types) 511 selected_PDU_boots = list(set(boot_list + boot_stack) 512 & set(boot_lists['PDU_reboot'])) 513 if len(selected_PDU_boots) > 0 and pdu_host == "": 514 error_message = "You have selected the following boots which" +\ 515 " require a PDU host but no value for pdu_host:\n" 516 error_message += gp.sprint_var(selected_PDU_boots) 517 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 518 BuiltIn().fail(gp.sprint_error(error_message)) 519 520 return 521 522 523def my_get_state(): 524 r""" 525 Get the system state plus a little bit of wrapping. 526 """ 527 528 global state 529 530 req_states = ['epoch_seconds'] + st.default_req_states 531 532 gp.qprint_timen("Getting system state.") 533 if test_mode: 534 state['epoch_seconds'] = int(time.time()) 535 else: 536 state = st.get_state(req_states=req_states, quiet=quiet) 537 gp.qprint_var(state) 538 539 540def valid_state(): 541 r""" 542 Verify that our state dictionary contains no blank values. If we don't get 543 valid state data, we cannot continue to work. 544 """ 545 546 if st.compare_states(state, st.invalid_state_match, 'or'): 547 error_message = "The state dictionary contains blank fields which" +\ 548 " is illegal.\n" + gp.sprint_var(state) 549 BuiltIn().fail(gp.sprint_error(error_message)) 550 551 552def select_boot(): 553 r""" 554 Select a boot test to be run based on our current state and return the 555 chosen boot type. 556 557 Description of arguments: 558 state The state of the machine. 559 """ 560 561 global transitional_boot_selected 562 global boot_stack 563 564 gp.qprint_timen("Selecting a boot test.") 565 566 if transitional_boot_selected and not boot_success: 567 prior_boot = next_boot 568 boot_candidate = boot_stack.pop() 569 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 570 + " transition to a valid state for '" + boot_candidate 571 + "' which was at the top of the boot_stack. Since" 572 + " the '" + next_boot + "' failed, the '" 573 + boot_candidate + "' has been removed from the stack" 574 + " to avoid and endless failure loop.") 575 if len(boot_stack) == 0: 576 return "" 577 578 my_get_state() 579 valid_state() 580 581 transitional_boot_selected = False 582 stack_popped = 0 583 if len(boot_stack) > 0: 584 stack_popped = 1 585 gp.qprint_dashes() 586 gp.qprint_var(boot_stack) 587 gp.qprint_dashes() 588 skip_boot_printed = 0 589 while len(boot_stack) > 0: 590 boot_candidate = boot_stack.pop() 591 if stack_mode == 'normal': 592 break 593 else: 594 if st.compare_states(state, boot_table[boot_candidate]['end']): 595 if not skip_boot_printed: 596 gp.qprint_var(stack_mode) 597 gp.qprintn() 598 gp.qprint_timen("Skipping the following boot tests" 599 + " which are unnecessary since their" 600 + " required end states match the" 601 + " current machine state:") 602 skip_boot_printed = 1 603 gp.qprint_var(boot_candidate) 604 boot_candidate = "" 605 if boot_candidate == "": 606 gp.qprint_dashes() 607 gp.qprint_var(boot_stack) 608 gp.qprint_dashes() 609 return boot_candidate 610 if st.compare_states(state, boot_table[boot_candidate]['start']): 611 gp.qprint_timen("The machine state is valid for a '" 612 + boot_candidate + "' boot test.") 613 gp.qprint_dashes() 614 gp.qprint_var(boot_stack) 615 gp.qprint_dashes() 616 return boot_candidate 617 else: 618 gp.qprint_timen("The machine state does not match the required" 619 + " starting state for a '" + boot_candidate 620 + "' boot test:") 621 gp.qprint_varx("boot_table_start_entry", 622 boot_table[boot_candidate]['start']) 623 boot_stack.append(boot_candidate) 624 transitional_boot_selected = True 625 popped_boot = boot_candidate 626 627 # Loop through your list selecting a boot_candidates 628 boot_candidates = [] 629 for boot_candidate in boot_list: 630 if st.compare_states(state, boot_table[boot_candidate]['start']): 631 if stack_popped: 632 if st.compare_states(boot_table[boot_candidate]['end'], 633 boot_table[popped_boot]['start']): 634 boot_candidates.append(boot_candidate) 635 else: 636 boot_candidates.append(boot_candidate) 637 638 if len(boot_candidates) == 0: 639 gp.qprint_timen("The user's boot list contained no boot tests" 640 + " which are valid for the current machine state.") 641 boot_candidate = default_power_on 642 if not st.compare_states(state, boot_table[default_power_on]['start']): 643 boot_candidate = default_power_off 644 boot_candidates.append(boot_candidate) 645 gp.qprint_timen("Using default '" + boot_candidate 646 + "' boot type to transition to valid state.") 647 648 gp.dprint_var(boot_candidates) 649 650 # Randomly select a boot from the candidate list. 651 boot = random.choice(boot_candidates) 652 653 return boot 654 655 656def print_defect_report(ffdc_file_list): 657 r""" 658 Print a defect report. 659 660 Description of argument(s): 661 ffdc_file_list A list of files which were collected by our ffdc functions. 662 """ 663 664 # Making deliberate choice to NOT run plug_in_setup(). We don't want 665 # ffdc_prefix updated. 666 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 667 call_point='ffdc_report', stop_on_plug_in_failure=0) 668 669 # Get additional header data which may have been created by ffdc plug-ins. 670 # Also, delete the individual header files to cleanup. 671 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 672 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 673 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 674 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 675 show_err=0) 676 677 # Get additional summary data which may have been created by ffdc plug-ins. 678 # Also, delete the individual header files to cleanup. 679 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 680 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 681 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 682 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 683 show_err=0) 684 685 # ffdc_list_file_path contains a list of any ffdc files created by plug- 686 # ins, etc. Read that data into a list. 687 try: 688 plug_in_ffdc_list = \ 689 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 690 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 691 except IOError: 692 plug_in_ffdc_list = [] 693 694 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 695 # in. Eliminate duplicates and sort the list. 696 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 697 698 if status_file_path != "": 699 ffdc_file_list.insert(0, status_file_path) 700 701 # Convert the list to a printable list. 702 printable_ffdc_file_list = "\n".join(ffdc_file_list) 703 704 # Open ffdc_file_list for writing. We will write a complete list of 705 # FFDC files to it for possible use by plug-ins like cp_stop_check. 706 ffdc_list_file = open(ffdc_list_file_path, 'w') 707 ffdc_list_file.write(printable_ffdc_file_list + "\n") 708 ffdc_list_file.close() 709 710 indent = 0 711 width = 90 712 linefeed = 1 713 char = "=" 714 715 gp.qprintn() 716 gp.qprint_dashes(indent, width, linefeed, char) 717 gp.qprintn("Copy this data to the defect:\n") 718 719 if len(more_header_info) > 0: 720 gp.qprintn(more_header_info) 721 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 722 openbmc_host_name, openbmc_ip, openbmc_username, 723 openbmc_password, rest_username, rest_password, ipmi_username, 724 ipmi_password, os_host, os_host_name, os_ip, os_username, 725 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 726 pdu_password, pdu_slot_no, openbmc_serial_host, 727 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 728 729 gp.qprintn() 730 print_boot_history(boot_history) 731 gp.qprintn() 732 gp.qprint_var(state) 733 gp.qprintn() 734 gp.qprintn("FFDC data files:") 735 gp.qprintn(printable_ffdc_file_list) 736 gp.qprintn() 737 738 if len(ffdc_summary_info) > 0: 739 gp.qprintn(ffdc_summary_info) 740 741 gp.qprint_dashes(indent, width, linefeed, char) 742 743 744def my_ffdc(): 745 r""" 746 Collect FFDC data. 747 """ 748 749 global state 750 751 plug_in_setup() 752 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 753 call_point='ffdc', stop_on_plug_in_failure=0) 754 755 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 756 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 757 + AUTOBOOT_FFDC_PREFIX 758 + " ffdc_function_list=" 759 + ffdc_function_list, ignore=1) 760 if status != 'PASS': 761 gp.qprint_error("Call to ffdc failed.\n") 762 if type(ffdc_file_list) is not list: 763 ffdc_file_list = [] 764 # Leave a record for caller that "soft" errors occurred. 765 soft_errors = 1 766 gpu.save_plug_in_value(soft_errors, pgm_name) 767 768 my_get_state() 769 770 print_defect_report(ffdc_file_list) 771 772 773def print_test_start_message(boot_keyword): 774 r""" 775 Print a message indicating what boot test is about to run. 776 777 Description of arguments: 778 boot_keyword The name of the boot which is to be run 779 (e.g. "BMC Power On"). 780 """ 781 782 global boot_history 783 global boot_start_time 784 785 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 786 787 # Set boot_start_time for use by plug-ins. 788 boot_start_time = doing_msg[1:33] 789 gp.qprint_var(boot_start_time) 790 791 gp.qprint(doing_msg) 792 793 update_boot_history(boot_history, doing_msg, max_boot_history) 794 795 796def stop_boot_test(signal_number=0, 797 frame=None): 798 r""" 799 Handle SIGUSR1 by aborting the boot test that is running. 800 801 Description of argument(s): 802 signal_number The signal number (should always be 10 for SIGUSR1). 803 frame The frame data. 804 """ 805 806 gp.qprintn() 807 gp.qprint_executing() 808 gp.lprint_executing() 809 810 # Restore original sigusr1 handler. 811 set_default_siguser1() 812 813 message = "The caller has asked that the boot test be stopped and marked" 814 message += " as a failure." 815 816 function_stack = gm.get_function_stack() 817 if "wait_state" in function_stack: 818 st.set_exit_wait_early_message(message) 819 else: 820 BuiltIn().fail(gp.sprint_error(message)) 821 822 823def run_boot(boot): 824 r""" 825 Run the specified boot. 826 827 Description of arguments: 828 boot The name of the boot test to be performed. 829 """ 830 831 global state 832 833 signal.signal(signal.SIGUSR1, stop_boot_test) 834 gp.qprint_timen("stop_boot_test is armed.") 835 836 print_test_start_message(boot) 837 838 plug_in_setup() 839 rc, shell_rc, failed_plug_in_name = \ 840 grpi.rprocess_plug_in_packages(call_point="pre_boot") 841 if rc != 0: 842 error_message = "Plug-in failed with non-zero return code.\n" +\ 843 gp.sprint_var(rc, fmt=gp.hexa()) 844 set_default_siguser1() 845 BuiltIn().fail(gp.sprint_error(error_message)) 846 847 if test_mode: 848 # In test mode, we'll pretend the boot worked by assigning its 849 # required end state to the default state value. 850 state = st.strip_anchor_state(boot_table[boot]['end']) 851 else: 852 # Assertion: We trust that the state data was made fresh by the 853 # caller. 854 855 gp.qprintn() 856 857 if boot_table[boot]['method_type'] == "keyword": 858 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 859 boot_table[boot]['method'], 860 quiet=quiet) 861 862 if boot_table[boot]['bmc_reboot']: 863 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 864 plug_in_setup() 865 rc, shell_rc, failed_plug_in_name = \ 866 grpi.rprocess_plug_in_packages(call_point="post_reboot") 867 if rc != 0: 868 error_message = "Plug-in failed with non-zero return code.\n" 869 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 870 set_default_siguser1() 871 BuiltIn().fail(gp.sprint_error(error_message)) 872 else: 873 match_state = st.anchor_state(state) 874 del match_state['epoch_seconds'] 875 # Wait for the state to change in any way. 876 st.wait_state(match_state, wait_time=state_change_timeout, 877 interval="10 seconds", invert=1) 878 879 gp.qprintn() 880 if boot_table[boot]['end']['chassis'] == "Off": 881 boot_timeout = power_off_timeout 882 else: 883 boot_timeout = power_on_timeout 884 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 885 interval="10 seconds") 886 887 plug_in_setup() 888 rc, shell_rc, failed_plug_in_name = \ 889 grpi.rprocess_plug_in_packages(call_point="post_boot") 890 if rc != 0: 891 error_message = "Plug-in failed with non-zero return code.\n" +\ 892 gp.sprint_var(rc, fmt=gp.hexa()) 893 set_default_siguser1() 894 BuiltIn().fail(gp.sprint_error(error_message)) 895 896 # Restore original sigusr1 handler. 897 set_default_siguser1() 898 899 900def test_loop_body(): 901 r""" 902 The main loop body for the loop in main_py. 903 904 Description of arguments: 905 boot_count The iteration number (starts at 1). 906 """ 907 908 global boot_count 909 global state 910 global next_boot 911 global boot_success 912 global boot_end_time 913 914 gp.qprintn() 915 916 next_boot = select_boot() 917 if next_boot == "": 918 return True 919 920 boot_count += 1 921 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 922 923 pre_boot_plug_in_setup() 924 925 cmd_buf = ["run_boot", next_boot] 926 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 927 if boot_status == "FAIL": 928 gp.qprint(msg) 929 930 gp.qprintn() 931 if boot_status == "PASS": 932 boot_success = 1 933 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 934 + "\" succeeded.") 935 else: 936 boot_success = 0 937 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 938 + "\" failed.") 939 940 # Set boot_end_time for use by plug-ins. 941 boot_end_time = completion_msg[1:33] 942 gp.qprint_var(boot_end_time) 943 944 gp.qprint(completion_msg) 945 946 boot_results.update(next_boot, boot_status) 947 948 plug_in_setup() 949 # NOTE: A post_test_case call point failure is NOT counted as a boot 950 # failure. 951 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 952 call_point='post_test_case', stop_on_plug_in_failure=0) 953 954 plug_in_setup() 955 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 956 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 957 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 958 if ffdc_check == "All" or\ 959 shell_rc == dump_ffdc_rc(): 960 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 961 if status != 'PASS': 962 gp.qprint_error("Call to my_ffdc failed.\n") 963 # Leave a record for caller that "soft" errors occurred. 964 soft_errors = 1 965 gpu.save_plug_in_value(soft_errors, pgm_name) 966 967 if delete_errlogs: 968 # We need to purge error logs between boots or they build up. 969 grk.run_key(delete_errlogs_cmd, ignore=1) 970 971 boot_results.print_report() 972 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 973 974 plug_in_setup() 975 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 976 call_point='stop_check', shell_rc=stop_test_rc(), 977 stop_on_non_zero_rc=1) 978 if shell_rc == stop_test_rc(): 979 message = "Stopping as requested by user.\n" 980 gp.qprint_time(message) 981 BuiltIn().fail(message) 982 983 # This should help prevent ConnectionErrors. 984 grk.run_key_u("Close All Connections") 985 986 return True 987 988 989def obmc_boot_test_teardown(): 990 r""" 991 Clean up after the main keyword. 992 """ 993 gp.qprint_executing() 994 995 if ga.psutil_imported: 996 ga.terminate_descendants() 997 998 if cp_setup_called: 999 plug_in_setup() 1000 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1001 call_point='cleanup', stop_on_plug_in_failure=0) 1002 1003 if 'boot_results_file_path' in globals(): 1004 # Save boot_results and boot_history objects to a file in case they are 1005 # needed again. 1006 gp.qprint_timen("Saving boot_results to the following path.") 1007 gp.qprint_var(boot_results_file_path) 1008 pickle.dump((boot_results, boot_history), 1009 open(boot_results_file_path, 'wb'), 1010 pickle.HIGHEST_PROTOCOL) 1011 1012 global save_stack 1013 # Restore any global values saved on the save_stack. 1014 for parm_name in main_func_parm_list: 1015 # Get the parm_value if it was saved on the stack. 1016 try: 1017 parm_value = save_stack.pop(parm_name) 1018 except BaseException: 1019 # If it was not saved, no further action is required. 1020 continue 1021 1022 # Restore the saved value. 1023 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1024 "}\", parm_value)" 1025 gp.dpissuing(cmd_buf) 1026 exec(cmd_buf) 1027 1028 gp.dprintn(save_stack.sprint_obj()) 1029 1030 1031def test_teardown(): 1032 r""" 1033 Clean up after this test case. 1034 """ 1035 1036 gp.qprintn() 1037 gp.qprint_executing() 1038 1039 if ga.psutil_imported: 1040 ga.terminate_descendants() 1041 1042 cmd_buf = ["Print Error", 1043 "A keyword timeout occurred ending this program.\n"] 1044 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1045 1046 if redfish_supported: 1047 redfish.logout() 1048 1049 gp.qprint_pgm_footer() 1050 1051 1052def post_stack(): 1053 r""" 1054 Process post_stack plug-in programs. 1055 """ 1056 1057 if not call_post_stack_plug: 1058 # The caller does not wish to have post_stack plug-in processing done. 1059 return 1060 1061 global boot_success 1062 1063 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1064 pre_boot_plug_in_setup() 1065 # For the purposes of the following plug-ins, mark the "boot" as a success. 1066 boot_success = 1 1067 plug_in_setup() 1068 rc, shell_rc, failed_plug_in_name, history =\ 1069 grpi.rprocess_plug_in_packages(call_point='post_stack', 1070 stop_on_plug_in_failure=0, 1071 return_history=True) 1072 for doing_msg in history: 1073 update_boot_history(boot_history, doing_msg, max_boot_history) 1074 if rc != 0: 1075 boot_success = 0 1076 1077 plug_in_setup() 1078 rc, shell_rc, failed_plug_in_name =\ 1079 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1080 shell_rc=dump_ffdc_rc(), 1081 stop_on_plug_in_failure=1, 1082 stop_on_non_zero_rc=1) 1083 if shell_rc == dump_ffdc_rc(): 1084 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1085 if status != 'PASS': 1086 gp.qprint_error("Call to my_ffdc failed.\n") 1087 # Leave a record for caller that "soft" errors occurred. 1088 soft_errors = 1 1089 gpu.save_plug_in_value(soft_errors, pgm_name) 1090 1091 plug_in_setup() 1092 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1093 call_point='stop_check', shell_rc=stop_test_rc(), 1094 stop_on_non_zero_rc=1) 1095 if shell_rc == stop_test_rc(): 1096 message = "Stopping as requested by user.\n" 1097 gp.qprint_time(message) 1098 BuiltIn().fail(message) 1099 1100 1101def obmc_boot_test_py(loc_boot_stack=None, 1102 loc_stack_mode=None, 1103 loc_quiet=None): 1104 r""" 1105 Do main program processing. 1106 """ 1107 1108 global save_stack 1109 1110 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1111 1112 gp.dprintn() 1113 # Process function parms. 1114 for parm_name in main_func_parm_list: 1115 # Get parm's value. 1116 parm_value = eval("loc_" + parm_name) 1117 gp.dpvars(parm_name, parm_value) 1118 1119 if parm_value is not None: 1120 # Save the global value on a stack. 1121 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1122 parm_name + "}\"), \"" + parm_name + "\")" 1123 gp.dpissuing(cmd_buf) 1124 exec(cmd_buf) 1125 1126 # Set the global value to the passed value. 1127 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1128 "}\", loc_" + parm_name + ")" 1129 gp.dpissuing(cmd_buf) 1130 exec(cmd_buf) 1131 1132 gp.dprintn(save_stack.sprint_obj()) 1133 1134 setup() 1135 1136 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1137 1138 if ffdc_only: 1139 gp.qprint_timen("Caller requested ffdc_only.") 1140 if do_pre_boot_plug_in_setup: 1141 pre_boot_plug_in_setup() 1142 grk.run_key_u("my_ffdc") 1143 return 1144 1145 if delete_errlogs: 1146 # Delete errlogs prior to doing any boot tests. 1147 grk.run_key(delete_errlogs_cmd, ignore=1) 1148 1149 # Process caller's boot_stack. 1150 while (len(boot_stack) > 0): 1151 test_loop_body() 1152 1153 gp.qprint_timen("Finished processing stack.") 1154 1155 post_stack() 1156 1157 # Process caller's boot_list. 1158 if len(boot_list) > 0: 1159 for ix in range(1, max_num_tests + 1): 1160 test_loop_body() 1161 1162 gp.qprint_timen("Completed all requested boot tests.") 1163 1164 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1165 new_fail = boot_fail - init_boot_fail 1166 if new_fail > boot_fail_threshold: 1167 error_message = "Boot failures exceed the boot failure" +\ 1168 " threshold:\n" +\ 1169 gp.sprint_var(new_fail) +\ 1170 gp.sprint_var(boot_fail_threshold) 1171 BuiltIn().fail(gp.sprint_error(error_message)) 1172