1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34 35base_path = os.path.dirname(os.path.dirname( 36 imp.find_module("gen_robot_print")[1])) +\ 37 os.sep 38sys.path.append(base_path + "extended/") 39import run_keyword as rk 40 41# Setting master_pid correctly influences the behavior of plug-ins like 42# DB_Logging 43program_pid = os.getpid() 44master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 45pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 46 47# Set up boot data structures. 48os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 49boot_table = create_boot_table(os_host=os_host) 50valid_boot_types = create_valid_boot_list(boot_table) 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 70if redfish_supported: 71 default_power_on = "Redfish Power On" 72 default_power_off = "Redfish Power Off" 73else: 74 default_power_on = "REST Power On" 75 default_power_off = "REST Power Off" 76boot_count = 0 77 78LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 79AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 80ffdc_prefix = AUTOBOOT_FFDC_PREFIX 81boot_start_time = "" 82boot_end_time = "" 83save_stack = vs.var_stack('save_stack') 84main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 85 86 87def dump_ffdc_rc(): 88 r""" 89 Return the constant dump ffdc test return code value. 90 91 When a plug-in call point program returns this value, it indicates that 92 this program should collect FFDC. 93 """ 94 95 return 0x00000200 96 97 98def stop_test_rc(): 99 r""" 100 Return the constant stop test return code value. 101 102 When a plug-in call point program returns this value, it indicates that 103 this program should stop running. 104 """ 105 106 return 0x00000200 107 108 109def process_host(host, 110 host_var_name=""): 111 r""" 112 Process a host by getting the associated host name and IP address and 113 setting them in global variables. 114 115 If the caller does not pass the host_var_name, this function will try to 116 figure out the name of the variable used by the caller for the host parm. 117 Callers are advised to explicitly specify the host_var_name when calling 118 with an exec command. In such cases, the get_arg_name cannot figure out 119 the host variable name. 120 121 This function will then create similar global variable names by 122 removing "_host" and appending "_host_name" or "_ip" to the host variable 123 name. 124 125 Example: 126 127 If a call is made like this: 128 process_host(openbmc_host) 129 130 Global variables openbmc_host_name and openbmc_ip will be set. 131 132 Description of argument(s): 133 host A host name or IP. The name of the variable used should 134 have a suffix of "_host". 135 host_var_name The name of the variable being used as the host parm. 136 """ 137 138 if host_var_name == "": 139 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 140 141 host_name_var_name = re.sub("host", "host_name", host_var_name) 142 ip_var_name = re.sub("host", "ip", host_var_name) 143 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 144 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 145 host + "')" 146 exec(cmd_buf) 147 148 149def process_pgm_parms(): 150 r""" 151 Process the program parameters by assigning them all to corresponding 152 globals. Also, set some global values that depend on program parameters. 153 """ 154 155 # Program parameter processing. 156 # Assign all program parms to python variables which are global to this 157 # module. 158 159 global parm_list 160 parm_list = BuiltIn().get_variable_value("${parm_list}") 161 # The following subset of parms should be processed as integers. 162 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 163 'boot_fail_threshold', 'delete_errlogs', 164 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 165 'test_mode', 'debug'] 166 for parm in parm_list: 167 if parm in int_list: 168 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 169 "}\", \"0\"))" 170 else: 171 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 172 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 173 gp.dpissuing(cmd_buf) 174 exec(cmd_buf) 175 if re.match(r".*_host$", parm): 176 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 177 exec(cmd_buf) 178 if re.match(r".*_password$", parm): 179 # Register the value of any parm whose name ends in _password. 180 # This will cause the print functions to replace passwords with 181 # asterisks in the output. 182 cmd_buf = "gp.register_passwords(" + parm + ")" 183 exec(cmd_buf) 184 185 global ffdc_dir_path_style 186 global boot_list 187 global boot_stack 188 global boot_results_file_path 189 global boot_results 190 global boot_history 191 global ffdc_list_file_path 192 global ffdc_report_list_path 193 global ffdc_summary_list_path 194 195 if ffdc_dir_path_style == "": 196 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 197 198 # Convert these program parms to lists for easier processing.. 199 boot_list = list(filter(None, boot_list.split(":"))) 200 boot_stack = list(filter(None, boot_stack.split(":"))) 201 202 cleanup_boot_results_file() 203 boot_results_file_path = create_boot_results_file_path(pgm_name, 204 openbmc_nickname, 205 master_pid) 206 207 if os.path.isfile(boot_results_file_path): 208 # We've been called before in this run so we'll load the saved 209 # boot_results and boot_history objects. 210 boot_results, boot_history =\ 211 pickle.load(open(boot_results_file_path, 'rb')) 212 else: 213 boot_results = boot_results(boot_table, boot_pass, boot_fail) 214 215 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 216 "/FFDC_FILE_LIST" 217 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 218 "/FFDC_REPORT_FILE_LIST" 219 220 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 221 "/FFDC_SUMMARY_FILE_LIST" 222 223 224def initial_plug_in_setup(): 225 r""" 226 Initialize all plug-in environment variables which do not change for the 227 duration of the program. 228 229 """ 230 231 global LOG_LEVEL 232 BuiltIn().set_log_level("NONE") 233 234 BuiltIn().set_global_variable("${master_pid}", master_pid) 235 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 236 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 237 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 238 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 239 ffdc_list_file_path) 240 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 241 ffdc_report_list_path) 242 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 243 ffdc_summary_list_path) 244 245 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 246 ffdc_dir_path_style) 247 BuiltIn().set_global_variable("${FFDC_CHECK}", 248 ffdc_check) 249 250 # For each program parameter, set the corresponding AUTOBOOT_ environment 251 # variable value. Also, set an AUTOBOOT_ environment variable for every 252 # element in additional_values. 253 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 254 "status_dir_path", "base_tool_dir_path", 255 "ffdc_list_file_path", "ffdc_report_list_path", 256 "ffdc_summary_list_path", "execdir"] 257 258 plug_in_vars = parm_list + additional_values 259 260 for var_name in plug_in_vars: 261 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 262 var_name = var_name.upper() 263 if var_value is None: 264 var_value = "" 265 os.environ["AUTOBOOT_" + var_name] = str(var_value) 266 267 BuiltIn().set_log_level(LOG_LEVEL) 268 269 # Make sure the ffdc list directory exists. 270 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 271 if not os.path.exists(ffdc_list_dir_path): 272 os.makedirs(ffdc_list_dir_path) 273 274 275def plug_in_setup(): 276 r""" 277 Initialize all changing plug-in environment variables for use by the 278 plug-in programs. 279 """ 280 281 global LOG_LEVEL 282 global test_really_running 283 284 BuiltIn().set_log_level("NONE") 285 286 boot_pass, boot_fail = boot_results.return_total_pass_fail() 287 if boot_pass > 1: 288 test_really_running = 1 289 else: 290 test_really_running = 0 291 292 BuiltIn().set_global_variable("${test_really_running}", 293 test_really_running) 294 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 295 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 296 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 297 BuiltIn().set_global_variable("${boot_success}", boot_success) 298 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 299 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 300 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 301 302 # For each program parameter, set the corresponding AUTOBOOT_ environment 303 # variable value. Also, set an AUTOBOOT_ environment variable for every 304 # element in additional_values. 305 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 306 "boot_fail", "test_really_running", "ffdc_prefix", 307 "boot_start_time", "boot_end_time"] 308 309 plug_in_vars = additional_values 310 311 for var_name in plug_in_vars: 312 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 313 var_name = var_name.upper() 314 if var_value is None: 315 var_value = "" 316 os.environ["AUTOBOOT_" + var_name] = str(var_value) 317 318 if debug: 319 shell_rc, out_buf = \ 320 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 321 322 BuiltIn().set_log_level(LOG_LEVEL) 323 324 325def pre_boot_plug_in_setup(): 326 327 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 328 try: 329 os.remove(ffdc_list_file_path) 330 except OSError: 331 pass 332 333 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 334 try: 335 os.remove(ffdc_report_list_path) 336 except OSError: 337 pass 338 339 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 340 try: 341 os.remove(ffdc_summary_list_path) 342 except OSError: 343 pass 344 345 global ffdc_prefix 346 347 seconds = time.time() 348 loc_time = time.localtime(seconds) 349 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 350 351 ffdc_prefix = openbmc_nickname + "." + time_string 352 353 354def default_sigusr1(signal_number=0, 355 frame=None): 356 r""" 357 Handle SIGUSR1 by doing nothing. 358 359 This function assists in debugging SIGUSR1 processing by printing messages 360 to stdout and to the log.html file. 361 362 Description of argument(s): 363 signal_number The signal number (should always be 10 for SIGUSR1). 364 frame The frame data. 365 """ 366 367 gp.qprintn() 368 gp.qprint_executing() 369 gp.lprint_executing() 370 371 372def set_default_siguser1(): 373 r""" 374 Set the default_sigusr1 function to be the SIGUSR1 handler. 375 """ 376 377 gp.qprintn() 378 gp.qprint_executing() 379 gp.lprint_executing() 380 signal.signal(signal.SIGUSR1, default_sigusr1) 381 382 383def setup(): 384 r""" 385 Do general program setup tasks. 386 """ 387 388 global cp_setup_called 389 global transitional_boot_selected 390 391 gp.qprintn() 392 393 set_default_siguser1() 394 transitional_boot_selected = False 395 396 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 397 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 398 # If we can't find process_plug_in_packages.py, ssh_pw or 399 # validate_plug_ins.py, then we don't have our repo bin in PATH. 400 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 401 + " ssh_pw validate_plug_ins.py", quiet=1, 402 print_output=0, show_err=0) 403 if shell_rc != 0: 404 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 405 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 406 if robot_pgm_dir_path not in sys.path: 407 sys.path.append(robot_pgm_dir_path) 408 PYTHONPATH = os.environ.get("PYTHONPATH", "") 409 if PYTHONPATH == "": 410 os.environ['PYTHONPATH'] = robot_pgm_dir_path 411 else: 412 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 413 414 validate_parms() 415 416 gp.qprint_pgm_header() 417 418 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 419 420 initial_plug_in_setup() 421 422 plug_in_setup() 423 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 424 call_point='setup') 425 if rc != 0: 426 error_message = "Plug-in setup failed.\n" 427 gp.print_error_report(error_message) 428 BuiltIn().fail(error_message) 429 # Setting cp_setup_called lets our Teardown know that it needs to call 430 # the cleanup plug-in call point. 431 cp_setup_called = 1 432 433 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 434 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 435 # FFDC_LOG_PATH is used by "FFDC" keyword. 436 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 437 438 # Also printed by FFDC. 439 global host_name 440 global host_ip 441 host = socket.gethostname() 442 host_name, host_ip = gm.get_host_name_ip(host) 443 444 gp.dprint_var(boot_table) 445 gp.dprint_var(boot_lists) 446 447 448def validate_parms(): 449 r""" 450 Validate all program parameters. 451 """ 452 453 process_pgm_parms() 454 455 gp.qprintn() 456 457 global openbmc_model 458 gv.valid_value(openbmc_host) 459 gv.valid_value(openbmc_username) 460 gv.valid_value(openbmc_password) 461 gv.valid_value(rest_username) 462 gv.valid_value(rest_password) 463 gv.valid_value(ipmi_username) 464 gv.valid_value(ipmi_password) 465 if os_host != "": 466 gv.valid_value(os_username) 467 gv.valid_value(os_password) 468 469 if pdu_host != "": 470 gv.valid_value(pdu_username) 471 gv.valid_value(pdu_password) 472 gv.valid_integer(pdu_slot_no) 473 if openbmc_serial_host != "": 474 gv.valid_integer(openbmc_serial_port) 475 if openbmc_model == "": 476 status, ret_values =\ 477 grk.run_key_u("Get BMC System Model") 478 openbmc_model = ret_values 479 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 480 gv.valid_value(openbmc_model) 481 gv.valid_integer(max_num_tests) 482 gv.valid_integer(boot_pass) 483 gv.valid_integer(boot_fail) 484 485 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 486 BuiltIn().set_global_variable("${plug_in_packages_list}", 487 plug_in_packages_list) 488 489 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 490 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 491 error_message = "You must provide either a value for either the" +\ 492 " boot_list or the boot_stack parm.\n" 493 BuiltIn().fail(gp.sprint_error(error_message)) 494 495 valid_boot_list(boot_list, valid_boot_types) 496 valid_boot_list(boot_stack, valid_boot_types) 497 498 selected_PDU_boots = list(set(boot_list + boot_stack) 499 & set(boot_lists['PDU_reboot'])) 500 501 if len(selected_PDU_boots) > 0 and pdu_host == "": 502 error_message = "You have selected the following boots which" +\ 503 " require a PDU host but no value for pdu_host:\n" 504 error_message += gp.sprint_var(selected_PDU_boots) 505 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 506 BuiltIn().fail(gp.sprint_error(error_message)) 507 508 return 509 510 511def my_get_state(): 512 r""" 513 Get the system state plus a little bit of wrapping. 514 """ 515 516 global state 517 518 req_states = ['epoch_seconds'] + st.default_req_states 519 520 gp.qprint_timen("Getting system state.") 521 if test_mode: 522 state['epoch_seconds'] = int(time.time()) 523 else: 524 state = st.get_state(req_states=req_states, quiet=quiet) 525 gp.qprint_var(state) 526 527 528def valid_state(): 529 r""" 530 Verify that our state dictionary contains no blank values. If we don't get 531 valid state data, we cannot continue to work. 532 """ 533 534 if st.compare_states(state, st.invalid_state_match, 'or'): 535 error_message = "The state dictionary contains blank fields which" +\ 536 " is illegal.\n" + gp.sprint_var(state) 537 BuiltIn().fail(gp.sprint_error(error_message)) 538 539 540def select_boot(): 541 r""" 542 Select a boot test to be run based on our current state and return the 543 chosen boot type. 544 545 Description of arguments: 546 state The state of the machine. 547 """ 548 549 global transitional_boot_selected 550 global boot_stack 551 552 gp.qprint_timen("Selecting a boot test.") 553 554 if transitional_boot_selected and not boot_success: 555 prior_boot = next_boot 556 boot_candidate = boot_stack.pop() 557 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 558 + " transition to a valid state for '" + boot_candidate 559 + "' which was at the top of the boot_stack. Since" 560 + " the '" + next_boot + "' failed, the '" 561 + boot_candidate + "' has been removed from the stack" 562 + " to avoid and endless failure loop.") 563 if len(boot_stack) == 0: 564 return "" 565 566 my_get_state() 567 valid_state() 568 569 transitional_boot_selected = False 570 stack_popped = 0 571 if len(boot_stack) > 0: 572 stack_popped = 1 573 gp.qprint_dashes() 574 gp.qprint_var(boot_stack) 575 gp.qprint_dashes() 576 skip_boot_printed = 0 577 while len(boot_stack) > 0: 578 boot_candidate = boot_stack.pop() 579 if stack_mode == 'normal': 580 break 581 else: 582 if st.compare_states(state, boot_table[boot_candidate]['end']): 583 if not skip_boot_printed: 584 gp.qprint_var(stack_mode) 585 gp.qprintn() 586 gp.qprint_timen("Skipping the following boot tests" 587 + " which are unnecessary since their" 588 + " required end states match the" 589 + " current machine state:") 590 skip_boot_printed = 1 591 gp.qprint_var(boot_candidate) 592 boot_candidate = "" 593 if boot_candidate == "": 594 gp.qprint_dashes() 595 gp.qprint_var(boot_stack) 596 gp.qprint_dashes() 597 return boot_candidate 598 if st.compare_states(state, boot_table[boot_candidate]['start']): 599 gp.qprint_timen("The machine state is valid for a '" 600 + boot_candidate + "' boot test.") 601 gp.qprint_dashes() 602 gp.qprint_var(boot_stack) 603 gp.qprint_dashes() 604 return boot_candidate 605 else: 606 gp.qprint_timen("The machine state does not match the required" 607 + " starting state for a '" + boot_candidate 608 + "' boot test:") 609 gp.qprint_varx("boot_table_start_entry", 610 boot_table[boot_candidate]['start']) 611 boot_stack.append(boot_candidate) 612 transitional_boot_selected = True 613 popped_boot = boot_candidate 614 615 # Loop through your list selecting a boot_candidates 616 boot_candidates = [] 617 for boot_candidate in boot_list: 618 if st.compare_states(state, boot_table[boot_candidate]['start']): 619 if stack_popped: 620 if st.compare_states(boot_table[boot_candidate]['end'], 621 boot_table[popped_boot]['start']): 622 boot_candidates.append(boot_candidate) 623 else: 624 boot_candidates.append(boot_candidate) 625 626 if len(boot_candidates) == 0: 627 gp.qprint_timen("The user's boot list contained no boot tests" 628 + " which are valid for the current machine state.") 629 boot_candidate = default_power_on 630 if not st.compare_states(state, boot_table[default_power_on]['start']): 631 boot_candidate = default_power_off 632 boot_candidates.append(boot_candidate) 633 gp.qprint_timen("Using default '" + boot_candidate 634 + "' boot type to transition to valid state.") 635 636 gp.dprint_var(boot_candidates) 637 638 # Randomly select a boot from the candidate list. 639 boot = random.choice(boot_candidates) 640 641 return boot 642 643 644def print_defect_report(ffdc_file_list): 645 r""" 646 Print a defect report. 647 648 Description of argument(s): 649 ffdc_file_list A list of files which were collected by our ffdc functions. 650 """ 651 652 # Making deliberate choice to NOT run plug_in_setup(). We don't want 653 # ffdc_prefix updated. 654 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 655 call_point='ffdc_report', stop_on_plug_in_failure=0) 656 657 # Get additional header data which may have been created by ffdc plug-ins. 658 # Also, delete the individual header files to cleanup. 659 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 660 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 661 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 662 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 663 show_err=0) 664 665 # Get additional summary data which may have been created by ffdc plug-ins. 666 # Also, delete the individual header files to cleanup. 667 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 668 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 669 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 670 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 671 show_err=0) 672 673 # ffdc_list_file_path contains a list of any ffdc files created by plug- 674 # ins, etc. Read that data into a list. 675 try: 676 plug_in_ffdc_list = \ 677 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 678 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 679 except IOError: 680 plug_in_ffdc_list = [] 681 682 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 683 # in. Eliminate duplicates and sort the list. 684 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 685 686 if status_file_path != "": 687 ffdc_file_list.insert(0, status_file_path) 688 689 # Convert the list to a printable list. 690 printable_ffdc_file_list = "\n".join(ffdc_file_list) 691 692 # Open ffdc_file_list for writing. We will write a complete list of 693 # FFDC files to it for possible use by plug-ins like cp_stop_check. 694 ffdc_list_file = open(ffdc_list_file_path, 'w') 695 ffdc_list_file.write(printable_ffdc_file_list + "\n") 696 ffdc_list_file.close() 697 698 indent = 0 699 width = 90 700 linefeed = 1 701 char = "=" 702 703 gp.qprintn() 704 gp.qprint_dashes(indent, width, linefeed, char) 705 gp.qprintn("Copy this data to the defect:\n") 706 707 if len(more_header_info) > 0: 708 gp.qprintn(more_header_info) 709 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 710 openbmc_host_name, openbmc_ip, openbmc_username, 711 openbmc_password, rest_username, rest_password, ipmi_username, 712 ipmi_password, os_host, os_host_name, os_ip, os_username, 713 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 714 pdu_password, pdu_slot_no, openbmc_serial_host, 715 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 716 717 gp.qprintn() 718 print_boot_history(boot_history) 719 gp.qprintn() 720 gp.qprint_var(state) 721 gp.qprintn() 722 gp.qprintn("FFDC data files:") 723 gp.qprintn(printable_ffdc_file_list) 724 gp.qprintn() 725 726 if len(ffdc_summary_info) > 0: 727 gp.qprintn(ffdc_summary_info) 728 729 gp.qprint_dashes(indent, width, linefeed, char) 730 731 732def my_ffdc(): 733 r""" 734 Collect FFDC data. 735 """ 736 737 global state 738 739 plug_in_setup() 740 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 741 call_point='ffdc', stop_on_plug_in_failure=0) 742 743 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 744 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 745 + AUTOBOOT_FFDC_PREFIX 746 + " ffdc_function_list=" 747 + ffdc_function_list, ignore=1) 748 if status != 'PASS': 749 gp.qprint_error("Call to ffdc failed.\n") 750 if type(ffdc_file_list) is not list: 751 ffdc_file_list = [] 752 # Leave a record for caller that "soft" errors occurred. 753 soft_errors = 1 754 gpu.save_plug_in_value(soft_errors, pgm_name) 755 756 my_get_state() 757 758 print_defect_report(ffdc_file_list) 759 760 761def print_test_start_message(boot_keyword): 762 r""" 763 Print a message indicating what boot test is about to run. 764 765 Description of arguments: 766 boot_keyword The name of the boot which is to be run 767 (e.g. "BMC Power On"). 768 """ 769 770 global boot_history 771 global boot_start_time 772 773 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 774 775 # Set boot_start_time for use by plug-ins. 776 boot_start_time = doing_msg[1:33] 777 gp.qprint_var(boot_start_time) 778 779 gp.qprint(doing_msg) 780 781 update_boot_history(boot_history, doing_msg, max_boot_history) 782 783 784def stop_boot_test(signal_number=0, 785 frame=None): 786 r""" 787 Handle SIGUSR1 by aborting the boot test that is running. 788 789 Description of argument(s): 790 signal_number The signal number (should always be 10 for SIGUSR1). 791 frame The frame data. 792 """ 793 794 gp.qprintn() 795 gp.qprint_executing() 796 gp.lprint_executing() 797 798 # Restore original sigusr1 handler. 799 set_default_siguser1() 800 801 message = "The caller has asked that the boot test be stopped and marked" 802 message += " as a failure." 803 804 function_stack = gm.get_function_stack() 805 if "wait_state" in function_stack: 806 st.set_exit_wait_early_message(message) 807 else: 808 BuiltIn().fail(gp.sprint_error(message)) 809 810 811def run_boot(boot): 812 r""" 813 Run the specified boot. 814 815 Description of arguments: 816 boot The name of the boot test to be performed. 817 """ 818 819 global state 820 821 signal.signal(signal.SIGUSR1, stop_boot_test) 822 gp.qprint_timen("stop_boot_test is armed.") 823 824 print_test_start_message(boot) 825 826 plug_in_setup() 827 rc, shell_rc, failed_plug_in_name = \ 828 grpi.rprocess_plug_in_packages(call_point="pre_boot") 829 if rc != 0: 830 error_message = "Plug-in failed with non-zero return code.\n" +\ 831 gp.sprint_var(rc, fmt=gp.hexa()) 832 set_default_siguser1() 833 BuiltIn().fail(gp.sprint_error(error_message)) 834 835 if test_mode: 836 # In test mode, we'll pretend the boot worked by assigning its 837 # required end state to the default state value. 838 state = st.strip_anchor_state(boot_table[boot]['end']) 839 else: 840 # Assertion: We trust that the state data was made fresh by the 841 # caller. 842 843 gp.qprintn() 844 845 if boot_table[boot]['method_type'] == "keyword": 846 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 847 boot_table[boot]['method'], 848 quiet=quiet) 849 850 if boot_table[boot]['bmc_reboot']: 851 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 852 plug_in_setup() 853 rc, shell_rc, failed_plug_in_name = \ 854 grpi.rprocess_plug_in_packages(call_point="post_reboot") 855 if rc != 0: 856 error_message = "Plug-in failed with non-zero return code.\n" 857 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 858 set_default_siguser1() 859 BuiltIn().fail(gp.sprint_error(error_message)) 860 else: 861 match_state = st.anchor_state(state) 862 del match_state['epoch_seconds'] 863 # Wait for the state to change in any way. 864 st.wait_state(match_state, wait_time=state_change_timeout, 865 interval="10 seconds", invert=1) 866 867 gp.qprintn() 868 if boot_table[boot]['end']['chassis'] == "Off": 869 boot_timeout = power_off_timeout 870 else: 871 boot_timeout = power_on_timeout 872 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 873 interval="10 seconds") 874 875 plug_in_setup() 876 rc, shell_rc, failed_plug_in_name = \ 877 grpi.rprocess_plug_in_packages(call_point="post_boot") 878 if rc != 0: 879 error_message = "Plug-in failed with non-zero return code.\n" +\ 880 gp.sprint_var(rc, fmt=gp.hexa()) 881 set_default_siguser1() 882 BuiltIn().fail(gp.sprint_error(error_message)) 883 884 # Restore original sigusr1 handler. 885 set_default_siguser1() 886 887 888def test_loop_body(): 889 r""" 890 The main loop body for the loop in main_py. 891 892 Description of arguments: 893 boot_count The iteration number (starts at 1). 894 """ 895 896 global boot_count 897 global state 898 global next_boot 899 global boot_success 900 global boot_end_time 901 902 gp.qprintn() 903 904 next_boot = select_boot() 905 if next_boot == "": 906 return True 907 908 boot_count += 1 909 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 910 911 pre_boot_plug_in_setup() 912 913 cmd_buf = ["run_boot", next_boot] 914 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 915 if boot_status == "FAIL": 916 gp.qprint(msg) 917 918 gp.qprintn() 919 if boot_status == "PASS": 920 boot_success = 1 921 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 922 + "\" succeeded.") 923 else: 924 boot_success = 0 925 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 926 + "\" failed.") 927 928 # Set boot_end_time for use by plug-ins. 929 boot_end_time = completion_msg[1:33] 930 gp.qprint_var(boot_end_time) 931 932 gp.qprint(completion_msg) 933 934 boot_results.update(next_boot, boot_status) 935 936 plug_in_setup() 937 # NOTE: A post_test_case call point failure is NOT counted as a boot 938 # failure. 939 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 940 call_point='post_test_case', stop_on_plug_in_failure=0) 941 942 plug_in_setup() 943 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 944 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 945 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 946 if ffdc_check == "All" or\ 947 shell_rc == dump_ffdc_rc(): 948 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 949 if status != 'PASS': 950 gp.qprint_error("Call to my_ffdc failed.\n") 951 # Leave a record for caller that "soft" errors occurred. 952 soft_errors = 1 953 gpu.save_plug_in_value(soft_errors, pgm_name) 954 955 if delete_errlogs: 956 # We need to purge error logs between boots or they build up. 957 grk.run_key("Delete Error logs", ignore=1) 958 959 boot_results.print_report() 960 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 961 962 plug_in_setup() 963 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 964 call_point='stop_check', shell_rc=stop_test_rc(), 965 stop_on_non_zero_rc=1) 966 if shell_rc == stop_test_rc(): 967 message = "Stopping as requested by user.\n" 968 gp.qprint_time(message) 969 BuiltIn().fail(message) 970 971 # This should help prevent ConnectionErrors. 972 grk.run_key_u("Close All Connections") 973 974 return True 975 976 977def obmc_boot_test_teardown(): 978 r""" 979 Clean up after the main keyword. 980 """ 981 gp.qprint_executing() 982 983 if ga.psutil_imported: 984 ga.terminate_descendants() 985 986 if cp_setup_called: 987 plug_in_setup() 988 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 989 call_point='cleanup', stop_on_plug_in_failure=0) 990 991 if 'boot_results_file_path' in globals(): 992 # Save boot_results and boot_history objects to a file in case they are 993 # needed again. 994 gp.qprint_timen("Saving boot_results to the following path.") 995 gp.qprint_var(boot_results_file_path) 996 pickle.dump((boot_results, boot_history), 997 open(boot_results_file_path, 'wb'), 998 pickle.HIGHEST_PROTOCOL) 999 1000 global save_stack 1001 # Restore any global values saved on the save_stack. 1002 for parm_name in main_func_parm_list: 1003 # Get the parm_value if it was saved on the stack. 1004 try: 1005 parm_value = save_stack.pop(parm_name) 1006 except BaseException: 1007 # If it was not saved, no further action is required. 1008 continue 1009 1010 # Restore the saved value. 1011 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1012 "}\", parm_value)" 1013 gp.dpissuing(cmd_buf) 1014 exec(cmd_buf) 1015 1016 gp.dprintn(save_stack.sprint_obj()) 1017 1018 1019def test_teardown(): 1020 r""" 1021 Clean up after this test case. 1022 """ 1023 1024 gp.qprintn() 1025 gp.qprint_executing() 1026 1027 if ga.psutil_imported: 1028 ga.terminate_descendants() 1029 1030 cmd_buf = ["Print Error", 1031 "A keyword timeout occurred ending this program.\n"] 1032 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1033 1034 gp.qprint_pgm_footer() 1035 1036 1037def post_stack(): 1038 r""" 1039 Process post_stack plug-in programs. 1040 """ 1041 1042 if not call_post_stack_plug: 1043 # The caller does not wish to have post_stack plug-in processing done. 1044 return 1045 1046 global boot_success 1047 1048 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1049 pre_boot_plug_in_setup() 1050 # For the purposes of the following plug-ins, mark the "boot" as a success. 1051 boot_success = 1 1052 plug_in_setup() 1053 rc, shell_rc, failed_plug_in_name, history =\ 1054 grpi.rprocess_plug_in_packages(call_point='post_stack', 1055 stop_on_plug_in_failure=0, 1056 return_history=True) 1057 for doing_msg in history: 1058 update_boot_history(boot_history, doing_msg, max_boot_history) 1059 if rc != 0: 1060 boot_success = 0 1061 1062 plug_in_setup() 1063 rc, shell_rc, failed_plug_in_name =\ 1064 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1065 shell_rc=dump_ffdc_rc(), 1066 stop_on_plug_in_failure=1, 1067 stop_on_non_zero_rc=1) 1068 if shell_rc == dump_ffdc_rc(): 1069 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1070 if status != 'PASS': 1071 gp.qprint_error("Call to my_ffdc failed.\n") 1072 # Leave a record for caller that "soft" errors occurred. 1073 soft_errors = 1 1074 gpu.save_plug_in_value(soft_errors, pgm_name) 1075 1076 plug_in_setup() 1077 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1078 call_point='stop_check', shell_rc=stop_test_rc(), 1079 stop_on_non_zero_rc=1) 1080 if shell_rc == stop_test_rc(): 1081 message = "Stopping as requested by user.\n" 1082 gp.qprint_time(message) 1083 BuiltIn().fail(message) 1084 1085 1086def obmc_boot_test_py(loc_boot_stack=None, 1087 loc_stack_mode=None, 1088 loc_quiet=None): 1089 r""" 1090 Do main program processing. 1091 """ 1092 1093 global save_stack 1094 1095 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1096 1097 gp.dprintn() 1098 # Process function parms. 1099 for parm_name in main_func_parm_list: 1100 # Get parm's value. 1101 parm_value = eval("loc_" + parm_name) 1102 gp.dpvars(parm_name, parm_value) 1103 1104 if parm_value is not None: 1105 # Save the global value on a stack. 1106 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1107 parm_name + "}\"), \"" + parm_name + "\")" 1108 gp.dpissuing(cmd_buf) 1109 exec(cmd_buf) 1110 1111 # Set the global value to the passed value. 1112 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1113 "}\", loc_" + parm_name + ")" 1114 gp.dpissuing(cmd_buf) 1115 exec(cmd_buf) 1116 1117 gp.dprintn(save_stack.sprint_obj()) 1118 1119 setup() 1120 1121 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1122 1123 if ffdc_only: 1124 gp.qprint_timen("Caller requested ffdc_only.") 1125 if do_pre_boot_plug_in_setup: 1126 pre_boot_plug_in_setup() 1127 grk.run_key_u("my_ffdc") 1128 return 1129 1130 # Process caller's boot_stack. 1131 while (len(boot_stack) > 0): 1132 test_loop_body() 1133 1134 gp.qprint_timen("Finished processing stack.") 1135 1136 post_stack() 1137 1138 # Process caller's boot_list. 1139 if len(boot_list) > 0: 1140 for ix in range(1, max_num_tests + 1): 1141 test_loop_body() 1142 1143 gp.qprint_timen("Completed all requested boot tests.") 1144 1145 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1146 new_fail = boot_fail - init_boot_fail 1147 if new_fail > boot_fail_threshold: 1148 error_message = "Boot failures exceed the boot failure" +\ 1149 " threshold:\n" +\ 1150 gp.sprint_var(new_fail) +\ 1151 gp.sprint_var(boot_fail_threshold) 1152 BuiltIn().fail(gp.sprint_error(error_message)) 1153