1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34 35base_path = os.path.dirname(os.path.dirname( 36 imp.find_module("gen_robot_print")[1])) +\ 37 os.sep 38sys.path.append(base_path + "extended/") 39import run_keyword as rk 40 41# Setting master_pid correctly influences the behavior of plug-ins like 42# DB_Logging 43program_pid = os.getpid() 44master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 45pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 46 47# Set up boot data structures. 48os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 49 50boot_lists = read_boot_lists() 51 52# The maximum number of entries that can be in the boot_history global variable. 53max_boot_history = 10 54boot_history = [] 55 56state = st.return_state_constant('default_state') 57cp_setup_called = 0 58next_boot = "" 59base_tool_dir_path = os.path.normpath(os.environ.get( 60 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 61 62ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 63boot_success = 0 64status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 65if status_dir_path != "": 66 status_dir_path = os.path.normpath(status_dir_path) + os.sep 67redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 68if redfish_supported: 69 default_power_on = "Redfish Power On" 70 default_power_off = "Redfish Power Off" 71else: 72 default_power_on = "REST Power On" 73 default_power_off = "REST Power Off" 74boot_count = 0 75 76LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 77AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 78ffdc_prefix = AUTOBOOT_FFDC_PREFIX 79boot_start_time = "" 80boot_end_time = "" 81save_stack = vs.var_stack('save_stack') 82main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 83 84 85def dump_ffdc_rc(): 86 r""" 87 Return the constant dump ffdc test return code value. 88 89 When a plug-in call point program returns this value, it indicates that 90 this program should collect FFDC. 91 """ 92 93 return 0x00000200 94 95 96def stop_test_rc(): 97 r""" 98 Return the constant stop test return code value. 99 100 When a plug-in call point program returns this value, it indicates that 101 this program should stop running. 102 """ 103 104 return 0x00000200 105 106 107def process_host(host, 108 host_var_name=""): 109 r""" 110 Process a host by getting the associated host name and IP address and 111 setting them in global variables. 112 113 If the caller does not pass the host_var_name, this function will try to 114 figure out the name of the variable used by the caller for the host parm. 115 Callers are advised to explicitly specify the host_var_name when calling 116 with an exec command. In such cases, the get_arg_name cannot figure out 117 the host variable name. 118 119 This function will then create similar global variable names by 120 removing "_host" and appending "_host_name" or "_ip" to the host variable 121 name. 122 123 Example: 124 125 If a call is made like this: 126 process_host(openbmc_host) 127 128 Global variables openbmc_host_name and openbmc_ip will be set. 129 130 Description of argument(s): 131 host A host name or IP. The name of the variable used should 132 have a suffix of "_host". 133 host_var_name The name of the variable being used as the host parm. 134 """ 135 136 if host_var_name == "": 137 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 138 139 host_name_var_name = re.sub("host", "host_name", host_var_name) 140 ip_var_name = re.sub("host", "ip", host_var_name) 141 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 142 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 143 host + "')" 144 exec(cmd_buf) 145 146 147def process_pgm_parms(): 148 r""" 149 Process the program parameters by assigning them all to corresponding 150 globals. Also, set some global values that depend on program parameters. 151 """ 152 153 # Program parameter processing. 154 # Assign all program parms to python variables which are global to this 155 # module. 156 157 global parm_list 158 parm_list = BuiltIn().get_variable_value("${parm_list}") 159 # The following subset of parms should be processed as integers. 160 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 161 'boot_fail_threshold', 'delete_errlogs', 162 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 163 'test_mode', 'debug'] 164 for parm in parm_list: 165 if parm in int_list: 166 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 167 "}\", \"0\"))" 168 else: 169 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 170 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 171 gp.dpissuing(cmd_buf) 172 exec(cmd_buf) 173 if re.match(r".*_host$", parm): 174 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 175 exec(cmd_buf) 176 if re.match(r".*_password$", parm): 177 # Register the value of any parm whose name ends in _password. 178 # This will cause the print functions to replace passwords with 179 # asterisks in the output. 180 cmd_buf = "gp.register_passwords(" + parm + ")" 181 exec(cmd_buf) 182 183 global ffdc_dir_path_style 184 global boot_list 185 global boot_stack 186 global boot_results_file_path 187 global boot_results 188 global boot_history 189 global ffdc_list_file_path 190 global ffdc_report_list_path 191 global ffdc_summary_list_path 192 global boot_table 193 global valid_boot_types 194 195 if ffdc_dir_path_style == "": 196 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 197 198 # Convert these program parms to lists for easier processing.. 199 boot_list = list(filter(None, boot_list.split(":"))) 200 boot_stack = list(filter(None, boot_stack.split(":"))) 201 202 boot_table = create_boot_table(boot_table_path, os_host=os_host) 203 valid_boot_types = create_valid_boot_list(boot_table) 204 205 cleanup_boot_results_file() 206 boot_results_file_path = create_boot_results_file_path(pgm_name, 207 openbmc_nickname, 208 master_pid) 209 210 if os.path.isfile(boot_results_file_path): 211 # We've been called before in this run so we'll load the saved 212 # boot_results and boot_history objects. 213 boot_results, boot_history =\ 214 pickle.load(open(boot_results_file_path, 'rb')) 215 else: 216 boot_results = boot_results(boot_table, boot_pass, boot_fail) 217 218 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 219 "/FFDC_FILE_LIST" 220 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 221 "/FFDC_REPORT_FILE_LIST" 222 223 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 224 "/FFDC_SUMMARY_FILE_LIST" 225 226 227def initial_plug_in_setup(): 228 r""" 229 Initialize all plug-in environment variables which do not change for the 230 duration of the program. 231 232 """ 233 234 global LOG_LEVEL 235 BuiltIn().set_log_level("NONE") 236 237 BuiltIn().set_global_variable("${master_pid}", master_pid) 238 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 239 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 240 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 241 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 242 ffdc_list_file_path) 243 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 244 ffdc_report_list_path) 245 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 246 ffdc_summary_list_path) 247 248 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 249 ffdc_dir_path_style) 250 BuiltIn().set_global_variable("${FFDC_CHECK}", 251 ffdc_check) 252 253 # For each program parameter, set the corresponding AUTOBOOT_ environment 254 # variable value. Also, set an AUTOBOOT_ environment variable for every 255 # element in additional_values. 256 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 257 "status_dir_path", "base_tool_dir_path", 258 "ffdc_list_file_path", "ffdc_report_list_path", 259 "ffdc_summary_list_path", "execdir"] 260 261 plug_in_vars = parm_list + additional_values 262 263 for var_name in plug_in_vars: 264 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 265 var_name = var_name.upper() 266 if var_value is None: 267 var_value = "" 268 os.environ["AUTOBOOT_" + var_name] = str(var_value) 269 270 BuiltIn().set_log_level(LOG_LEVEL) 271 272 # Make sure the ffdc list directory exists. 273 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 274 if not os.path.exists(ffdc_list_dir_path): 275 os.makedirs(ffdc_list_dir_path) 276 277 278def plug_in_setup(): 279 r""" 280 Initialize all changing plug-in environment variables for use by the 281 plug-in programs. 282 """ 283 284 global LOG_LEVEL 285 global test_really_running 286 287 BuiltIn().set_log_level("NONE") 288 289 boot_pass, boot_fail = boot_results.return_total_pass_fail() 290 if boot_pass > 1: 291 test_really_running = 1 292 else: 293 test_really_running = 0 294 295 BuiltIn().set_global_variable("${test_really_running}", 296 test_really_running) 297 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 298 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 299 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 300 BuiltIn().set_global_variable("${boot_success}", boot_success) 301 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 302 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 303 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 304 305 # For each program parameter, set the corresponding AUTOBOOT_ environment 306 # variable value. Also, set an AUTOBOOT_ environment variable for every 307 # element in additional_values. 308 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 309 "boot_fail", "test_really_running", "ffdc_prefix", 310 "boot_start_time", "boot_end_time"] 311 312 plug_in_vars = additional_values 313 314 for var_name in plug_in_vars: 315 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 316 var_name = var_name.upper() 317 if var_value is None: 318 var_value = "" 319 os.environ["AUTOBOOT_" + var_name] = str(var_value) 320 321 if debug: 322 shell_rc, out_buf = \ 323 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 324 325 BuiltIn().set_log_level(LOG_LEVEL) 326 327 328def pre_boot_plug_in_setup(): 329 330 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 331 try: 332 os.remove(ffdc_list_file_path) 333 except OSError: 334 pass 335 336 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 337 try: 338 os.remove(ffdc_report_list_path) 339 except OSError: 340 pass 341 342 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 343 try: 344 os.remove(ffdc_summary_list_path) 345 except OSError: 346 pass 347 348 global ffdc_prefix 349 350 seconds = time.time() 351 loc_time = time.localtime(seconds) 352 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 353 354 ffdc_prefix = openbmc_nickname + "." + time_string 355 356 357def default_sigusr1(signal_number=0, 358 frame=None): 359 r""" 360 Handle SIGUSR1 by doing nothing. 361 362 This function assists in debugging SIGUSR1 processing by printing messages 363 to stdout and to the log.html file. 364 365 Description of argument(s): 366 signal_number The signal number (should always be 10 for SIGUSR1). 367 frame The frame data. 368 """ 369 370 gp.qprintn() 371 gp.qprint_executing() 372 gp.lprint_executing() 373 374 375def set_default_siguser1(): 376 r""" 377 Set the default_sigusr1 function to be the SIGUSR1 handler. 378 """ 379 380 gp.qprintn() 381 gp.qprint_executing() 382 gp.lprint_executing() 383 signal.signal(signal.SIGUSR1, default_sigusr1) 384 385 386def setup(): 387 r""" 388 Do general program setup tasks. 389 """ 390 391 global cp_setup_called 392 global transitional_boot_selected 393 394 gp.qprintn() 395 396 set_default_siguser1() 397 transitional_boot_selected = False 398 399 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 400 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 401 # If we can't find process_plug_in_packages.py, ssh_pw or 402 # validate_plug_ins.py, then we don't have our repo bin in PATH. 403 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 404 + " ssh_pw validate_plug_ins.py", quiet=1, 405 print_output=0, show_err=0) 406 if shell_rc != 0: 407 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 408 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 409 if robot_pgm_dir_path not in sys.path: 410 sys.path.append(robot_pgm_dir_path) 411 PYTHONPATH = os.environ.get("PYTHONPATH", "") 412 if PYTHONPATH == "": 413 os.environ['PYTHONPATH'] = robot_pgm_dir_path 414 else: 415 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 416 417 validate_parms() 418 419 gp.qprint_pgm_header() 420 421 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 422 423 initial_plug_in_setup() 424 425 plug_in_setup() 426 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 427 call_point='setup') 428 if rc != 0: 429 error_message = "Plug-in setup failed.\n" 430 gp.print_error_report(error_message) 431 BuiltIn().fail(error_message) 432 # Setting cp_setup_called lets our Teardown know that it needs to call 433 # the cleanup plug-in call point. 434 cp_setup_called = 1 435 436 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 437 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 438 # FFDC_LOG_PATH is used by "FFDC" keyword. 439 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 440 441 # Also printed by FFDC. 442 global host_name 443 global host_ip 444 host = socket.gethostname() 445 host_name, host_ip = gm.get_host_name_ip(host) 446 447 gp.dprint_var(boot_table) 448 gp.dprint_var(boot_lists) 449 450 451def validate_parms(): 452 r""" 453 Validate all program parameters. 454 """ 455 456 process_pgm_parms() 457 458 gp.qprintn() 459 460 global openbmc_model 461 gv.valid_value(openbmc_host) 462 gv.valid_value(openbmc_username) 463 gv.valid_value(openbmc_password) 464 gv.valid_value(rest_username) 465 gv.valid_value(rest_password) 466 gv.valid_value(ipmi_username) 467 gv.valid_value(ipmi_password) 468 if os_host != "": 469 gv.valid_value(os_username) 470 gv.valid_value(os_password) 471 472 if pdu_host != "": 473 gv.valid_value(pdu_username) 474 gv.valid_value(pdu_password) 475 gv.valid_integer(pdu_slot_no) 476 if openbmc_serial_host != "": 477 gv.valid_integer(openbmc_serial_port) 478 if openbmc_model == "": 479 status, ret_values =\ 480 grk.run_key_u("Get BMC System Model") 481 openbmc_model = ret_values 482 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 483 gv.valid_value(openbmc_model) 484 gv.valid_integer(max_num_tests) 485 gv.valid_integer(boot_pass) 486 gv.valid_integer(boot_fail) 487 488 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 489 BuiltIn().set_global_variable("${plug_in_packages_list}", 490 plug_in_packages_list) 491 492 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 493 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 494 error_message = "You must provide either a value for either the" +\ 495 " boot_list or the boot_stack parm.\n" 496 BuiltIn().fail(gp.sprint_error(error_message)) 497 498 valid_boot_list(boot_list, valid_boot_types) 499 valid_boot_list(boot_stack, valid_boot_types) 500 501 selected_PDU_boots = list(set(boot_list + boot_stack) 502 & set(boot_lists['PDU_reboot'])) 503 504 if len(selected_PDU_boots) > 0 and pdu_host == "": 505 error_message = "You have selected the following boots which" +\ 506 " require a PDU host but no value for pdu_host:\n" 507 error_message += gp.sprint_var(selected_PDU_boots) 508 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 509 BuiltIn().fail(gp.sprint_error(error_message)) 510 511 return 512 513 514def my_get_state(): 515 r""" 516 Get the system state plus a little bit of wrapping. 517 """ 518 519 global state 520 521 req_states = ['epoch_seconds'] + st.default_req_states 522 523 gp.qprint_timen("Getting system state.") 524 if test_mode: 525 state['epoch_seconds'] = int(time.time()) 526 else: 527 state = st.get_state(req_states=req_states, quiet=quiet) 528 gp.qprint_var(state) 529 530 531def valid_state(): 532 r""" 533 Verify that our state dictionary contains no blank values. If we don't get 534 valid state data, we cannot continue to work. 535 """ 536 537 if st.compare_states(state, st.invalid_state_match, 'or'): 538 error_message = "The state dictionary contains blank fields which" +\ 539 " is illegal.\n" + gp.sprint_var(state) 540 BuiltIn().fail(gp.sprint_error(error_message)) 541 542 543def select_boot(): 544 r""" 545 Select a boot test to be run based on our current state and return the 546 chosen boot type. 547 548 Description of arguments: 549 state The state of the machine. 550 """ 551 552 global transitional_boot_selected 553 global boot_stack 554 555 gp.qprint_timen("Selecting a boot test.") 556 557 if transitional_boot_selected and not boot_success: 558 prior_boot = next_boot 559 boot_candidate = boot_stack.pop() 560 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 561 + " transition to a valid state for '" + boot_candidate 562 + "' which was at the top of the boot_stack. Since" 563 + " the '" + next_boot + "' failed, the '" 564 + boot_candidate + "' has been removed from the stack" 565 + " to avoid and endless failure loop.") 566 if len(boot_stack) == 0: 567 return "" 568 569 my_get_state() 570 valid_state() 571 572 transitional_boot_selected = False 573 stack_popped = 0 574 if len(boot_stack) > 0: 575 stack_popped = 1 576 gp.qprint_dashes() 577 gp.qprint_var(boot_stack) 578 gp.qprint_dashes() 579 skip_boot_printed = 0 580 while len(boot_stack) > 0: 581 boot_candidate = boot_stack.pop() 582 if stack_mode == 'normal': 583 break 584 else: 585 if st.compare_states(state, boot_table[boot_candidate]['end']): 586 if not skip_boot_printed: 587 gp.qprint_var(stack_mode) 588 gp.qprintn() 589 gp.qprint_timen("Skipping the following boot tests" 590 + " which are unnecessary since their" 591 + " required end states match the" 592 + " current machine state:") 593 skip_boot_printed = 1 594 gp.qprint_var(boot_candidate) 595 boot_candidate = "" 596 if boot_candidate == "": 597 gp.qprint_dashes() 598 gp.qprint_var(boot_stack) 599 gp.qprint_dashes() 600 return boot_candidate 601 if st.compare_states(state, boot_table[boot_candidate]['start']): 602 gp.qprint_timen("The machine state is valid for a '" 603 + boot_candidate + "' boot test.") 604 gp.qprint_dashes() 605 gp.qprint_var(boot_stack) 606 gp.qprint_dashes() 607 return boot_candidate 608 else: 609 gp.qprint_timen("The machine state does not match the required" 610 + " starting state for a '" + boot_candidate 611 + "' boot test:") 612 gp.qprint_varx("boot_table_start_entry", 613 boot_table[boot_candidate]['start']) 614 boot_stack.append(boot_candidate) 615 transitional_boot_selected = True 616 popped_boot = boot_candidate 617 618 # Loop through your list selecting a boot_candidates 619 boot_candidates = [] 620 for boot_candidate in boot_list: 621 if st.compare_states(state, boot_table[boot_candidate]['start']): 622 if stack_popped: 623 if st.compare_states(boot_table[boot_candidate]['end'], 624 boot_table[popped_boot]['start']): 625 boot_candidates.append(boot_candidate) 626 else: 627 boot_candidates.append(boot_candidate) 628 629 if len(boot_candidates) == 0: 630 gp.qprint_timen("The user's boot list contained no boot tests" 631 + " which are valid for the current machine state.") 632 boot_candidate = default_power_on 633 if not st.compare_states(state, boot_table[default_power_on]['start']): 634 boot_candidate = default_power_off 635 boot_candidates.append(boot_candidate) 636 gp.qprint_timen("Using default '" + boot_candidate 637 + "' boot type to transition to valid state.") 638 639 gp.dprint_var(boot_candidates) 640 641 # Randomly select a boot from the candidate list. 642 boot = random.choice(boot_candidates) 643 644 return boot 645 646 647def print_defect_report(ffdc_file_list): 648 r""" 649 Print a defect report. 650 651 Description of argument(s): 652 ffdc_file_list A list of files which were collected by our ffdc functions. 653 """ 654 655 # Making deliberate choice to NOT run plug_in_setup(). We don't want 656 # ffdc_prefix updated. 657 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 658 call_point='ffdc_report', stop_on_plug_in_failure=0) 659 660 # Get additional header data which may have been created by ffdc plug-ins. 661 # Also, delete the individual header files to cleanup. 662 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 663 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 664 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 665 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 666 show_err=0) 667 668 # Get additional summary data which may have been created by ffdc plug-ins. 669 # Also, delete the individual header files to cleanup. 670 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 671 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 672 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 673 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 674 show_err=0) 675 676 # ffdc_list_file_path contains a list of any ffdc files created by plug- 677 # ins, etc. Read that data into a list. 678 try: 679 plug_in_ffdc_list = \ 680 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 681 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 682 except IOError: 683 plug_in_ffdc_list = [] 684 685 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 686 # in. Eliminate duplicates and sort the list. 687 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 688 689 if status_file_path != "": 690 ffdc_file_list.insert(0, status_file_path) 691 692 # Convert the list to a printable list. 693 printable_ffdc_file_list = "\n".join(ffdc_file_list) 694 695 # Open ffdc_file_list for writing. We will write a complete list of 696 # FFDC files to it for possible use by plug-ins like cp_stop_check. 697 ffdc_list_file = open(ffdc_list_file_path, 'w') 698 ffdc_list_file.write(printable_ffdc_file_list + "\n") 699 ffdc_list_file.close() 700 701 indent = 0 702 width = 90 703 linefeed = 1 704 char = "=" 705 706 gp.qprintn() 707 gp.qprint_dashes(indent, width, linefeed, char) 708 gp.qprintn("Copy this data to the defect:\n") 709 710 if len(more_header_info) > 0: 711 gp.qprintn(more_header_info) 712 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 713 openbmc_host_name, openbmc_ip, openbmc_username, 714 openbmc_password, rest_username, rest_password, ipmi_username, 715 ipmi_password, os_host, os_host_name, os_ip, os_username, 716 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 717 pdu_password, pdu_slot_no, openbmc_serial_host, 718 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 719 720 gp.qprintn() 721 print_boot_history(boot_history) 722 gp.qprintn() 723 gp.qprint_var(state) 724 gp.qprintn() 725 gp.qprintn("FFDC data files:") 726 gp.qprintn(printable_ffdc_file_list) 727 gp.qprintn() 728 729 if len(ffdc_summary_info) > 0: 730 gp.qprintn(ffdc_summary_info) 731 732 gp.qprint_dashes(indent, width, linefeed, char) 733 734 735def my_ffdc(): 736 r""" 737 Collect FFDC data. 738 """ 739 740 global state 741 742 plug_in_setup() 743 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 744 call_point='ffdc', stop_on_plug_in_failure=0) 745 746 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 747 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 748 + AUTOBOOT_FFDC_PREFIX 749 + " ffdc_function_list=" 750 + ffdc_function_list, ignore=1) 751 if status != 'PASS': 752 gp.qprint_error("Call to ffdc failed.\n") 753 if type(ffdc_file_list) is not list: 754 ffdc_file_list = [] 755 # Leave a record for caller that "soft" errors occurred. 756 soft_errors = 1 757 gpu.save_plug_in_value(soft_errors, pgm_name) 758 759 my_get_state() 760 761 print_defect_report(ffdc_file_list) 762 763 764def print_test_start_message(boot_keyword): 765 r""" 766 Print a message indicating what boot test is about to run. 767 768 Description of arguments: 769 boot_keyword The name of the boot which is to be run 770 (e.g. "BMC Power On"). 771 """ 772 773 global boot_history 774 global boot_start_time 775 776 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 777 778 # Set boot_start_time for use by plug-ins. 779 boot_start_time = doing_msg[1:33] 780 gp.qprint_var(boot_start_time) 781 782 gp.qprint(doing_msg) 783 784 update_boot_history(boot_history, doing_msg, max_boot_history) 785 786 787def stop_boot_test(signal_number=0, 788 frame=None): 789 r""" 790 Handle SIGUSR1 by aborting the boot test that is running. 791 792 Description of argument(s): 793 signal_number The signal number (should always be 10 for SIGUSR1). 794 frame The frame data. 795 """ 796 797 gp.qprintn() 798 gp.qprint_executing() 799 gp.lprint_executing() 800 801 # Restore original sigusr1 handler. 802 set_default_siguser1() 803 804 message = "The caller has asked that the boot test be stopped and marked" 805 message += " as a failure." 806 807 function_stack = gm.get_function_stack() 808 if "wait_state" in function_stack: 809 st.set_exit_wait_early_message(message) 810 else: 811 BuiltIn().fail(gp.sprint_error(message)) 812 813 814def run_boot(boot): 815 r""" 816 Run the specified boot. 817 818 Description of arguments: 819 boot The name of the boot test to be performed. 820 """ 821 822 global state 823 824 signal.signal(signal.SIGUSR1, stop_boot_test) 825 gp.qprint_timen("stop_boot_test is armed.") 826 827 print_test_start_message(boot) 828 829 plug_in_setup() 830 rc, shell_rc, failed_plug_in_name = \ 831 grpi.rprocess_plug_in_packages(call_point="pre_boot") 832 if rc != 0: 833 error_message = "Plug-in failed with non-zero return code.\n" +\ 834 gp.sprint_var(rc, fmt=gp.hexa()) 835 set_default_siguser1() 836 BuiltIn().fail(gp.sprint_error(error_message)) 837 838 if test_mode: 839 # In test mode, we'll pretend the boot worked by assigning its 840 # required end state to the default state value. 841 state = st.strip_anchor_state(boot_table[boot]['end']) 842 else: 843 # Assertion: We trust that the state data was made fresh by the 844 # caller. 845 846 gp.qprintn() 847 848 if boot_table[boot]['method_type'] == "keyword": 849 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 850 boot_table[boot]['method'], 851 quiet=quiet) 852 853 if boot_table[boot]['bmc_reboot']: 854 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 855 plug_in_setup() 856 rc, shell_rc, failed_plug_in_name = \ 857 grpi.rprocess_plug_in_packages(call_point="post_reboot") 858 if rc != 0: 859 error_message = "Plug-in failed with non-zero return code.\n" 860 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 861 set_default_siguser1() 862 BuiltIn().fail(gp.sprint_error(error_message)) 863 else: 864 match_state = st.anchor_state(state) 865 del match_state['epoch_seconds'] 866 # Wait for the state to change in any way. 867 st.wait_state(match_state, wait_time=state_change_timeout, 868 interval="10 seconds", invert=1) 869 870 gp.qprintn() 871 if boot_table[boot]['end']['chassis'] == "Off": 872 boot_timeout = power_off_timeout 873 else: 874 boot_timeout = power_on_timeout 875 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 876 interval="10 seconds") 877 878 plug_in_setup() 879 rc, shell_rc, failed_plug_in_name = \ 880 grpi.rprocess_plug_in_packages(call_point="post_boot") 881 if rc != 0: 882 error_message = "Plug-in failed with non-zero return code.\n" +\ 883 gp.sprint_var(rc, fmt=gp.hexa()) 884 set_default_siguser1() 885 BuiltIn().fail(gp.sprint_error(error_message)) 886 887 # Restore original sigusr1 handler. 888 set_default_siguser1() 889 890 891def test_loop_body(): 892 r""" 893 The main loop body for the loop in main_py. 894 895 Description of arguments: 896 boot_count The iteration number (starts at 1). 897 """ 898 899 global boot_count 900 global state 901 global next_boot 902 global boot_success 903 global boot_end_time 904 905 gp.qprintn() 906 907 next_boot = select_boot() 908 if next_boot == "": 909 return True 910 911 boot_count += 1 912 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 913 914 pre_boot_plug_in_setup() 915 916 cmd_buf = ["run_boot", next_boot] 917 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 918 if boot_status == "FAIL": 919 gp.qprint(msg) 920 921 gp.qprintn() 922 if boot_status == "PASS": 923 boot_success = 1 924 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 925 + "\" succeeded.") 926 else: 927 boot_success = 0 928 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 929 + "\" failed.") 930 931 # Set boot_end_time for use by plug-ins. 932 boot_end_time = completion_msg[1:33] 933 gp.qprint_var(boot_end_time) 934 935 gp.qprint(completion_msg) 936 937 boot_results.update(next_boot, boot_status) 938 939 plug_in_setup() 940 # NOTE: A post_test_case call point failure is NOT counted as a boot 941 # failure. 942 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 943 call_point='post_test_case', stop_on_plug_in_failure=0) 944 945 plug_in_setup() 946 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 947 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 948 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 949 if ffdc_check == "All" or\ 950 shell_rc == dump_ffdc_rc(): 951 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 952 if status != 'PASS': 953 gp.qprint_error("Call to my_ffdc failed.\n") 954 # Leave a record for caller that "soft" errors occurred. 955 soft_errors = 1 956 gpu.save_plug_in_value(soft_errors, pgm_name) 957 958 if delete_errlogs: 959 # We need to purge error logs between boots or they build up. 960 grk.run_key("Delete Error logs", ignore=1) 961 962 boot_results.print_report() 963 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 964 965 plug_in_setup() 966 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 967 call_point='stop_check', shell_rc=stop_test_rc(), 968 stop_on_non_zero_rc=1) 969 if shell_rc == stop_test_rc(): 970 message = "Stopping as requested by user.\n" 971 gp.qprint_time(message) 972 BuiltIn().fail(message) 973 974 # This should help prevent ConnectionErrors. 975 grk.run_key_u("Close All Connections") 976 977 return True 978 979 980def obmc_boot_test_teardown(): 981 r""" 982 Clean up after the main keyword. 983 """ 984 gp.qprint_executing() 985 986 if ga.psutil_imported: 987 ga.terminate_descendants() 988 989 if cp_setup_called: 990 plug_in_setup() 991 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 992 call_point='cleanup', stop_on_plug_in_failure=0) 993 994 if 'boot_results_file_path' in globals(): 995 # Save boot_results and boot_history objects to a file in case they are 996 # needed again. 997 gp.qprint_timen("Saving boot_results to the following path.") 998 gp.qprint_var(boot_results_file_path) 999 pickle.dump((boot_results, boot_history), 1000 open(boot_results_file_path, 'wb'), 1001 pickle.HIGHEST_PROTOCOL) 1002 1003 global save_stack 1004 # Restore any global values saved on the save_stack. 1005 for parm_name in main_func_parm_list: 1006 # Get the parm_value if it was saved on the stack. 1007 try: 1008 parm_value = save_stack.pop(parm_name) 1009 except BaseException: 1010 # If it was not saved, no further action is required. 1011 continue 1012 1013 # Restore the saved value. 1014 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1015 "}\", parm_value)" 1016 gp.dpissuing(cmd_buf) 1017 exec(cmd_buf) 1018 1019 gp.dprintn(save_stack.sprint_obj()) 1020 1021 1022def test_teardown(): 1023 r""" 1024 Clean up after this test case. 1025 """ 1026 1027 gp.qprintn() 1028 gp.qprint_executing() 1029 1030 if ga.psutil_imported: 1031 ga.terminate_descendants() 1032 1033 cmd_buf = ["Print Error", 1034 "A keyword timeout occurred ending this program.\n"] 1035 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1036 1037 gp.qprint_pgm_footer() 1038 1039 1040def post_stack(): 1041 r""" 1042 Process post_stack plug-in programs. 1043 """ 1044 1045 if not call_post_stack_plug: 1046 # The caller does not wish to have post_stack plug-in processing done. 1047 return 1048 1049 global boot_success 1050 1051 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1052 pre_boot_plug_in_setup() 1053 # For the purposes of the following plug-ins, mark the "boot" as a success. 1054 boot_success = 1 1055 plug_in_setup() 1056 rc, shell_rc, failed_plug_in_name, history =\ 1057 grpi.rprocess_plug_in_packages(call_point='post_stack', 1058 stop_on_plug_in_failure=0, 1059 return_history=True) 1060 for doing_msg in history: 1061 update_boot_history(boot_history, doing_msg, max_boot_history) 1062 if rc != 0: 1063 boot_success = 0 1064 1065 plug_in_setup() 1066 rc, shell_rc, failed_plug_in_name =\ 1067 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1068 shell_rc=dump_ffdc_rc(), 1069 stop_on_plug_in_failure=1, 1070 stop_on_non_zero_rc=1) 1071 if shell_rc == dump_ffdc_rc(): 1072 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1073 if status != 'PASS': 1074 gp.qprint_error("Call to my_ffdc failed.\n") 1075 # Leave a record for caller that "soft" errors occurred. 1076 soft_errors = 1 1077 gpu.save_plug_in_value(soft_errors, pgm_name) 1078 1079 plug_in_setup() 1080 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1081 call_point='stop_check', shell_rc=stop_test_rc(), 1082 stop_on_non_zero_rc=1) 1083 if shell_rc == stop_test_rc(): 1084 message = "Stopping as requested by user.\n" 1085 gp.qprint_time(message) 1086 BuiltIn().fail(message) 1087 1088 1089def obmc_boot_test_py(loc_boot_stack=None, 1090 loc_stack_mode=None, 1091 loc_quiet=None): 1092 r""" 1093 Do main program processing. 1094 """ 1095 1096 global save_stack 1097 1098 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1099 1100 gp.dprintn() 1101 # Process function parms. 1102 for parm_name in main_func_parm_list: 1103 # Get parm's value. 1104 parm_value = eval("loc_" + parm_name) 1105 gp.dpvars(parm_name, parm_value) 1106 1107 if parm_value is not None: 1108 # Save the global value on a stack. 1109 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1110 parm_name + "}\"), \"" + parm_name + "\")" 1111 gp.dpissuing(cmd_buf) 1112 exec(cmd_buf) 1113 1114 # Set the global value to the passed value. 1115 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1116 "}\", loc_" + parm_name + ")" 1117 gp.dpissuing(cmd_buf) 1118 exec(cmd_buf) 1119 1120 gp.dprintn(save_stack.sprint_obj()) 1121 1122 setup() 1123 1124 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1125 1126 if ffdc_only: 1127 gp.qprint_timen("Caller requested ffdc_only.") 1128 if do_pre_boot_plug_in_setup: 1129 pre_boot_plug_in_setup() 1130 grk.run_key_u("my_ffdc") 1131 return 1132 1133 # Process caller's boot_stack. 1134 while (len(boot_stack) > 0): 1135 test_loop_body() 1136 1137 gp.qprint_timen("Finished processing stack.") 1138 1139 post_stack() 1140 1141 # Process caller's boot_list. 1142 if len(boot_list) > 0: 1143 for ix in range(1, max_num_tests + 1): 1144 test_loop_body() 1145 1146 gp.qprint_timen("Completed all requested boot tests.") 1147 1148 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1149 new_fail = boot_fail - init_boot_fail 1150 if new_fail > boot_fail_threshold: 1151 error_message = "Boot failures exceed the boot failure" +\ 1152 " threshold:\n" +\ 1153 gp.sprint_var(new_fail) +\ 1154 gp.sprint_var(boot_fail_threshold) 1155 BuiltIn().fail(gp.sprint_error(error_message)) 1156