1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34 35base_path = os.path.dirname(os.path.dirname( 36 imp.find_module("gen_robot_print")[1])) +\ 37 os.sep 38sys.path.append(base_path + "extended/") 39import run_keyword as rk 40 41# Setting master_pid correctly influences the behavior of plug-ins like 42# DB_Logging 43program_pid = os.getpid() 44master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 45pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 46 47# Set up boot data structures. 48os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 49 50boot_lists = read_boot_lists() 51 52# The maximum number of entries that can be in the boot_history global variable. 53max_boot_history = 10 54boot_history = [] 55 56state = st.return_state_constant('default_state') 57cp_setup_called = 0 58next_boot = "" 59base_tool_dir_path = os.path.normpath(os.environ.get( 60 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 61 62ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 63boot_success = 0 64status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 65if status_dir_path != "": 66 status_dir_path = os.path.normpath(status_dir_path) + os.sep 67redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 68if redfish_supported: 69 default_power_on = "Redfish Power On" 70 default_power_off = "Redfish Power Off" 71 delete_errlogs_cmd = "Delete Error Logs" 72 # TODO: delete_errlogs_cmd="Redfish Purge Event Log" 73else: 74 default_power_on = "REST Power On" 75 default_power_off = "REST Power Off" 76 delete_errlogs_cmd = "Delete Error Logs" 77boot_count = 0 78 79LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 80AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 81ffdc_prefix = AUTOBOOT_FFDC_PREFIX 82boot_start_time = "" 83boot_end_time = "" 84save_stack = vs.var_stack('save_stack') 85main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 86 87 88def dump_ffdc_rc(): 89 r""" 90 Return the constant dump ffdc test return code value. 91 92 When a plug-in call point program returns this value, it indicates that 93 this program should collect FFDC. 94 """ 95 96 return 0x00000200 97 98 99def stop_test_rc(): 100 r""" 101 Return the constant stop test return code value. 102 103 When a plug-in call point program returns this value, it indicates that 104 this program should stop running. 105 """ 106 107 return 0x00000200 108 109 110def process_host(host, 111 host_var_name=""): 112 r""" 113 Process a host by getting the associated host name and IP address and 114 setting them in global variables. 115 116 If the caller does not pass the host_var_name, this function will try to 117 figure out the name of the variable used by the caller for the host parm. 118 Callers are advised to explicitly specify the host_var_name when calling 119 with an exec command. In such cases, the get_arg_name cannot figure out 120 the host variable name. 121 122 This function will then create similar global variable names by 123 removing "_host" and appending "_host_name" or "_ip" to the host variable 124 name. 125 126 Example: 127 128 If a call is made like this: 129 process_host(openbmc_host) 130 131 Global variables openbmc_host_name and openbmc_ip will be set. 132 133 Description of argument(s): 134 host A host name or IP. The name of the variable used should 135 have a suffix of "_host". 136 host_var_name The name of the variable being used as the host parm. 137 """ 138 139 if host_var_name == "": 140 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 141 142 host_name_var_name = re.sub("host", "host_name", host_var_name) 143 ip_var_name = re.sub("host", "ip", host_var_name) 144 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 145 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 146 host + "')" 147 exec(cmd_buf) 148 149 150def process_pgm_parms(): 151 r""" 152 Process the program parameters by assigning them all to corresponding 153 globals. Also, set some global values that depend on program parameters. 154 """ 155 156 # Program parameter processing. 157 # Assign all program parms to python variables which are global to this 158 # module. 159 160 global parm_list 161 parm_list = BuiltIn().get_variable_value("${parm_list}") 162 # The following subset of parms should be processed as integers. 163 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 164 'boot_fail_threshold', 'delete_errlogs', 165 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 166 'test_mode', 'debug'] 167 for parm in parm_list: 168 if parm in int_list: 169 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 170 "}\", \"0\"))" 171 else: 172 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 173 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 174 gp.dpissuing(cmd_buf) 175 exec(cmd_buf) 176 if re.match(r".*_host$", parm): 177 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 178 exec(cmd_buf) 179 if re.match(r".*_password$", parm): 180 # Register the value of any parm whose name ends in _password. 181 # This will cause the print functions to replace passwords with 182 # asterisks in the output. 183 cmd_buf = "gp.register_passwords(" + parm + ")" 184 exec(cmd_buf) 185 186 global ffdc_dir_path_style 187 global boot_list 188 global boot_stack 189 global boot_results_file_path 190 global boot_results 191 global boot_history 192 global ffdc_list_file_path 193 global ffdc_report_list_path 194 global ffdc_summary_list_path 195 global boot_table 196 global valid_boot_types 197 198 if ffdc_dir_path_style == "": 199 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 200 201 # Convert these program parms to lists for easier processing.. 202 boot_list = list(filter(None, boot_list.split(":"))) 203 boot_stack = list(filter(None, boot_stack.split(":"))) 204 205 boot_table = create_boot_table(boot_table_path, os_host=os_host) 206 valid_boot_types = create_valid_boot_list(boot_table) 207 208 cleanup_boot_results_file() 209 boot_results_file_path = create_boot_results_file_path(pgm_name, 210 openbmc_nickname, 211 master_pid) 212 213 if os.path.isfile(boot_results_file_path): 214 # We've been called before in this run so we'll load the saved 215 # boot_results and boot_history objects. 216 boot_results, boot_history =\ 217 pickle.load(open(boot_results_file_path, 'rb')) 218 else: 219 boot_results = boot_results(boot_table, boot_pass, boot_fail) 220 221 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 222 "/FFDC_FILE_LIST" 223 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 224 "/FFDC_REPORT_FILE_LIST" 225 226 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 227 "/FFDC_SUMMARY_FILE_LIST" 228 229 230def initial_plug_in_setup(): 231 r""" 232 Initialize all plug-in environment variables which do not change for the 233 duration of the program. 234 235 """ 236 237 global LOG_LEVEL 238 BuiltIn().set_log_level("NONE") 239 240 BuiltIn().set_global_variable("${master_pid}", master_pid) 241 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 242 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 243 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 244 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 245 ffdc_list_file_path) 246 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 247 ffdc_report_list_path) 248 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 249 ffdc_summary_list_path) 250 251 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 252 ffdc_dir_path_style) 253 BuiltIn().set_global_variable("${FFDC_CHECK}", 254 ffdc_check) 255 256 # For each program parameter, set the corresponding AUTOBOOT_ environment 257 # variable value. Also, set an AUTOBOOT_ environment variable for every 258 # element in additional_values. 259 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 260 "status_dir_path", "base_tool_dir_path", 261 "ffdc_list_file_path", "ffdc_report_list_path", 262 "ffdc_summary_list_path", "execdir"] 263 264 plug_in_vars = parm_list + additional_values 265 266 for var_name in plug_in_vars: 267 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 268 var_name = var_name.upper() 269 if var_value is None: 270 var_value = "" 271 os.environ["AUTOBOOT_" + var_name] = str(var_value) 272 273 BuiltIn().set_log_level(LOG_LEVEL) 274 275 # Make sure the ffdc list directory exists. 276 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 277 if not os.path.exists(ffdc_list_dir_path): 278 os.makedirs(ffdc_list_dir_path) 279 280 281def plug_in_setup(): 282 r""" 283 Initialize all changing plug-in environment variables for use by the 284 plug-in programs. 285 """ 286 287 global LOG_LEVEL 288 global test_really_running 289 290 BuiltIn().set_log_level("NONE") 291 292 boot_pass, boot_fail = boot_results.return_total_pass_fail() 293 if boot_pass > 1: 294 test_really_running = 1 295 else: 296 test_really_running = 0 297 298 BuiltIn().set_global_variable("${test_really_running}", 299 test_really_running) 300 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 301 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 302 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 303 BuiltIn().set_global_variable("${boot_success}", boot_success) 304 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 305 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 306 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 307 308 # For each program parameter, set the corresponding AUTOBOOT_ environment 309 # variable value. Also, set an AUTOBOOT_ environment variable for every 310 # element in additional_values. 311 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 312 "boot_fail", "test_really_running", "ffdc_prefix", 313 "boot_start_time", "boot_end_time"] 314 315 plug_in_vars = additional_values 316 317 for var_name in plug_in_vars: 318 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 319 var_name = var_name.upper() 320 if var_value is None: 321 var_value = "" 322 os.environ["AUTOBOOT_" + var_name] = str(var_value) 323 324 if debug: 325 shell_rc, out_buf = \ 326 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 327 328 BuiltIn().set_log_level(LOG_LEVEL) 329 330 331def pre_boot_plug_in_setup(): 332 333 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 334 try: 335 os.remove(ffdc_list_file_path) 336 except OSError: 337 pass 338 339 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 340 try: 341 os.remove(ffdc_report_list_path) 342 except OSError: 343 pass 344 345 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 346 try: 347 os.remove(ffdc_summary_list_path) 348 except OSError: 349 pass 350 351 global ffdc_prefix 352 353 seconds = time.time() 354 loc_time = time.localtime(seconds) 355 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 356 357 ffdc_prefix = openbmc_nickname + "." + time_string 358 359 360def default_sigusr1(signal_number=0, 361 frame=None): 362 r""" 363 Handle SIGUSR1 by doing nothing. 364 365 This function assists in debugging SIGUSR1 processing by printing messages 366 to stdout and to the log.html file. 367 368 Description of argument(s): 369 signal_number The signal number (should always be 10 for SIGUSR1). 370 frame The frame data. 371 """ 372 373 gp.qprintn() 374 gp.qprint_executing() 375 gp.lprint_executing() 376 377 378def set_default_siguser1(): 379 r""" 380 Set the default_sigusr1 function to be the SIGUSR1 handler. 381 """ 382 383 gp.qprintn() 384 gp.qprint_executing() 385 gp.lprint_executing() 386 signal.signal(signal.SIGUSR1, default_sigusr1) 387 388 389def setup(): 390 r""" 391 Do general program setup tasks. 392 """ 393 394 global cp_setup_called 395 global transitional_boot_selected 396 397 gp.qprintn() 398 399 set_default_siguser1() 400 transitional_boot_selected = False 401 402 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 403 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 404 # If we can't find process_plug_in_packages.py, ssh_pw or 405 # validate_plug_ins.py, then we don't have our repo bin in PATH. 406 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 407 + " ssh_pw validate_plug_ins.py", quiet=1, 408 print_output=0, show_err=0) 409 if shell_rc != 0: 410 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 411 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 412 if robot_pgm_dir_path not in sys.path: 413 sys.path.append(robot_pgm_dir_path) 414 PYTHONPATH = os.environ.get("PYTHONPATH", "") 415 if PYTHONPATH == "": 416 os.environ['PYTHONPATH'] = robot_pgm_dir_path 417 else: 418 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 419 420 validate_parms() 421 422 gp.qprint_pgm_header() 423 424 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 425 426 initial_plug_in_setup() 427 428 plug_in_setup() 429 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 430 call_point='setup') 431 if rc != 0: 432 error_message = "Plug-in setup failed.\n" 433 gp.print_error_report(error_message) 434 BuiltIn().fail(error_message) 435 # Setting cp_setup_called lets our Teardown know that it needs to call 436 # the cleanup plug-in call point. 437 cp_setup_called = 1 438 439 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 440 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 441 # FFDC_LOG_PATH is used by "FFDC" keyword. 442 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 443 444 # Also printed by FFDC. 445 global host_name 446 global host_ip 447 host = socket.gethostname() 448 host_name, host_ip = gm.get_host_name_ip(host) 449 450 gp.dprint_var(boot_table) 451 gp.dprint_var(boot_lists) 452 453 454def validate_parms(): 455 r""" 456 Validate all program parameters. 457 """ 458 459 process_pgm_parms() 460 461 gp.qprintn() 462 463 global openbmc_model 464 if openbmc_model == "": 465 status, ret_values =\ 466 grk.run_key_u("Get BMC System Model") 467 openbmc_model = ret_values 468 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 469 gv.set_exit_on_error(True) 470 gv.valid_value(openbmc_host) 471 gv.valid_value(openbmc_username) 472 gv.valid_value(openbmc_password) 473 gv.valid_value(rest_username) 474 gv.valid_value(rest_password) 475 gv.valid_value(ipmi_username) 476 gv.valid_value(ipmi_password) 477 if os_host != "": 478 gv.valid_value(os_username) 479 gv.valid_value(os_password) 480 if pdu_host != "": 481 gv.valid_value(pdu_username) 482 gv.valid_value(pdu_password) 483 gv.valid_integer(pdu_slot_no) 484 if openbmc_serial_host != "": 485 gv.valid_integer(openbmc_serial_port) 486 gv.valid_value(openbmc_model) 487 gv.valid_integer(max_num_tests) 488 gv.valid_integer(boot_pass) 489 gv.valid_integer(boot_fail) 490 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 491 BuiltIn().set_global_variable("${plug_in_packages_list}", 492 plug_in_packages_list) 493 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 494 gv.set_exit_on_error(False) 495 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 496 error_message = "You must provide either a value for either the" +\ 497 " boot_list or the boot_stack parm.\n" 498 BuiltIn().fail(gp.sprint_error(error_message)) 499 valid_boot_list(boot_list, valid_boot_types) 500 valid_boot_list(boot_stack, valid_boot_types) 501 selected_PDU_boots = list(set(boot_list + boot_stack) 502 & set(boot_lists['PDU_reboot'])) 503 if len(selected_PDU_boots) > 0 and pdu_host == "": 504 error_message = "You have selected the following boots which" +\ 505 " require a PDU host but no value for pdu_host:\n" 506 error_message += gp.sprint_var(selected_PDU_boots) 507 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 508 BuiltIn().fail(gp.sprint_error(error_message)) 509 510 return 511 512 513def my_get_state(): 514 r""" 515 Get the system state plus a little bit of wrapping. 516 """ 517 518 global state 519 520 req_states = ['epoch_seconds'] + st.default_req_states 521 522 gp.qprint_timen("Getting system state.") 523 if test_mode: 524 state['epoch_seconds'] = int(time.time()) 525 else: 526 state = st.get_state(req_states=req_states, quiet=quiet) 527 gp.qprint_var(state) 528 529 530def valid_state(): 531 r""" 532 Verify that our state dictionary contains no blank values. If we don't get 533 valid state data, we cannot continue to work. 534 """ 535 536 if st.compare_states(state, st.invalid_state_match, 'or'): 537 error_message = "The state dictionary contains blank fields which" +\ 538 " is illegal.\n" + gp.sprint_var(state) 539 BuiltIn().fail(gp.sprint_error(error_message)) 540 541 542def select_boot(): 543 r""" 544 Select a boot test to be run based on our current state and return the 545 chosen boot type. 546 547 Description of arguments: 548 state The state of the machine. 549 """ 550 551 global transitional_boot_selected 552 global boot_stack 553 554 gp.qprint_timen("Selecting a boot test.") 555 556 if transitional_boot_selected and not boot_success: 557 prior_boot = next_boot 558 boot_candidate = boot_stack.pop() 559 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 560 + " transition to a valid state for '" + boot_candidate 561 + "' which was at the top of the boot_stack. Since" 562 + " the '" + next_boot + "' failed, the '" 563 + boot_candidate + "' has been removed from the stack" 564 + " to avoid and endless failure loop.") 565 if len(boot_stack) == 0: 566 return "" 567 568 my_get_state() 569 valid_state() 570 571 transitional_boot_selected = False 572 stack_popped = 0 573 if len(boot_stack) > 0: 574 stack_popped = 1 575 gp.qprint_dashes() 576 gp.qprint_var(boot_stack) 577 gp.qprint_dashes() 578 skip_boot_printed = 0 579 while len(boot_stack) > 0: 580 boot_candidate = boot_stack.pop() 581 if stack_mode == 'normal': 582 break 583 else: 584 if st.compare_states(state, boot_table[boot_candidate]['end']): 585 if not skip_boot_printed: 586 gp.qprint_var(stack_mode) 587 gp.qprintn() 588 gp.qprint_timen("Skipping the following boot tests" 589 + " which are unnecessary since their" 590 + " required end states match the" 591 + " current machine state:") 592 skip_boot_printed = 1 593 gp.qprint_var(boot_candidate) 594 boot_candidate = "" 595 if boot_candidate == "": 596 gp.qprint_dashes() 597 gp.qprint_var(boot_stack) 598 gp.qprint_dashes() 599 return boot_candidate 600 if st.compare_states(state, boot_table[boot_candidate]['start']): 601 gp.qprint_timen("The machine state is valid for a '" 602 + boot_candidate + "' boot test.") 603 gp.qprint_dashes() 604 gp.qprint_var(boot_stack) 605 gp.qprint_dashes() 606 return boot_candidate 607 else: 608 gp.qprint_timen("The machine state does not match the required" 609 + " starting state for a '" + boot_candidate 610 + "' boot test:") 611 gp.qprint_varx("boot_table_start_entry", 612 boot_table[boot_candidate]['start']) 613 boot_stack.append(boot_candidate) 614 transitional_boot_selected = True 615 popped_boot = boot_candidate 616 617 # Loop through your list selecting a boot_candidates 618 boot_candidates = [] 619 for boot_candidate in boot_list: 620 if st.compare_states(state, boot_table[boot_candidate]['start']): 621 if stack_popped: 622 if st.compare_states(boot_table[boot_candidate]['end'], 623 boot_table[popped_boot]['start']): 624 boot_candidates.append(boot_candidate) 625 else: 626 boot_candidates.append(boot_candidate) 627 628 if len(boot_candidates) == 0: 629 gp.qprint_timen("The user's boot list contained no boot tests" 630 + " which are valid for the current machine state.") 631 boot_candidate = default_power_on 632 if not st.compare_states(state, boot_table[default_power_on]['start']): 633 boot_candidate = default_power_off 634 boot_candidates.append(boot_candidate) 635 gp.qprint_timen("Using default '" + boot_candidate 636 + "' boot type to transition to valid state.") 637 638 gp.dprint_var(boot_candidates) 639 640 # Randomly select a boot from the candidate list. 641 boot = random.choice(boot_candidates) 642 643 return boot 644 645 646def print_defect_report(ffdc_file_list): 647 r""" 648 Print a defect report. 649 650 Description of argument(s): 651 ffdc_file_list A list of files which were collected by our ffdc functions. 652 """ 653 654 # Making deliberate choice to NOT run plug_in_setup(). We don't want 655 # ffdc_prefix updated. 656 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 657 call_point='ffdc_report', stop_on_plug_in_failure=0) 658 659 # Get additional header data which may have been created by ffdc plug-ins. 660 # Also, delete the individual header files to cleanup. 661 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 662 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 663 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 664 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 665 show_err=0) 666 667 # Get additional summary data which may have been created by ffdc plug-ins. 668 # Also, delete the individual header files to cleanup. 669 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 670 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 671 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 672 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 673 show_err=0) 674 675 # ffdc_list_file_path contains a list of any ffdc files created by plug- 676 # ins, etc. Read that data into a list. 677 try: 678 plug_in_ffdc_list = \ 679 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 680 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 681 except IOError: 682 plug_in_ffdc_list = [] 683 684 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 685 # in. Eliminate duplicates and sort the list. 686 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 687 688 if status_file_path != "": 689 ffdc_file_list.insert(0, status_file_path) 690 691 # Convert the list to a printable list. 692 printable_ffdc_file_list = "\n".join(ffdc_file_list) 693 694 # Open ffdc_file_list for writing. We will write a complete list of 695 # FFDC files to it for possible use by plug-ins like cp_stop_check. 696 ffdc_list_file = open(ffdc_list_file_path, 'w') 697 ffdc_list_file.write(printable_ffdc_file_list + "\n") 698 ffdc_list_file.close() 699 700 indent = 0 701 width = 90 702 linefeed = 1 703 char = "=" 704 705 gp.qprintn() 706 gp.qprint_dashes(indent, width, linefeed, char) 707 gp.qprintn("Copy this data to the defect:\n") 708 709 if len(more_header_info) > 0: 710 gp.qprintn(more_header_info) 711 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 712 openbmc_host_name, openbmc_ip, openbmc_username, 713 openbmc_password, rest_username, rest_password, ipmi_username, 714 ipmi_password, os_host, os_host_name, os_ip, os_username, 715 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 716 pdu_password, pdu_slot_no, openbmc_serial_host, 717 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 718 719 gp.qprintn() 720 print_boot_history(boot_history) 721 gp.qprintn() 722 gp.qprint_var(state) 723 gp.qprintn() 724 gp.qprintn("FFDC data files:") 725 gp.qprintn(printable_ffdc_file_list) 726 gp.qprintn() 727 728 if len(ffdc_summary_info) > 0: 729 gp.qprintn(ffdc_summary_info) 730 731 gp.qprint_dashes(indent, width, linefeed, char) 732 733 734def my_ffdc(): 735 r""" 736 Collect FFDC data. 737 """ 738 739 global state 740 741 plug_in_setup() 742 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 743 call_point='ffdc', stop_on_plug_in_failure=0) 744 745 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 746 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 747 + AUTOBOOT_FFDC_PREFIX 748 + " ffdc_function_list=" 749 + ffdc_function_list, ignore=1) 750 if status != 'PASS': 751 gp.qprint_error("Call to ffdc failed.\n") 752 if type(ffdc_file_list) is not list: 753 ffdc_file_list = [] 754 # Leave a record for caller that "soft" errors occurred. 755 soft_errors = 1 756 gpu.save_plug_in_value(soft_errors, pgm_name) 757 758 my_get_state() 759 760 print_defect_report(ffdc_file_list) 761 762 763def print_test_start_message(boot_keyword): 764 r""" 765 Print a message indicating what boot test is about to run. 766 767 Description of arguments: 768 boot_keyword The name of the boot which is to be run 769 (e.g. "BMC Power On"). 770 """ 771 772 global boot_history 773 global boot_start_time 774 775 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 776 777 # Set boot_start_time for use by plug-ins. 778 boot_start_time = doing_msg[1:33] 779 gp.qprint_var(boot_start_time) 780 781 gp.qprint(doing_msg) 782 783 update_boot_history(boot_history, doing_msg, max_boot_history) 784 785 786def stop_boot_test(signal_number=0, 787 frame=None): 788 r""" 789 Handle SIGUSR1 by aborting the boot test that is running. 790 791 Description of argument(s): 792 signal_number The signal number (should always be 10 for SIGUSR1). 793 frame The frame data. 794 """ 795 796 gp.qprintn() 797 gp.qprint_executing() 798 gp.lprint_executing() 799 800 # Restore original sigusr1 handler. 801 set_default_siguser1() 802 803 message = "The caller has asked that the boot test be stopped and marked" 804 message += " as a failure." 805 806 function_stack = gm.get_function_stack() 807 if "wait_state" in function_stack: 808 st.set_exit_wait_early_message(message) 809 else: 810 BuiltIn().fail(gp.sprint_error(message)) 811 812 813def run_boot(boot): 814 r""" 815 Run the specified boot. 816 817 Description of arguments: 818 boot The name of the boot test to be performed. 819 """ 820 821 global state 822 823 signal.signal(signal.SIGUSR1, stop_boot_test) 824 gp.qprint_timen("stop_boot_test is armed.") 825 826 print_test_start_message(boot) 827 828 plug_in_setup() 829 rc, shell_rc, failed_plug_in_name = \ 830 grpi.rprocess_plug_in_packages(call_point="pre_boot") 831 if rc != 0: 832 error_message = "Plug-in failed with non-zero return code.\n" +\ 833 gp.sprint_var(rc, fmt=gp.hexa()) 834 set_default_siguser1() 835 BuiltIn().fail(gp.sprint_error(error_message)) 836 837 if test_mode: 838 # In test mode, we'll pretend the boot worked by assigning its 839 # required end state to the default state value. 840 state = st.strip_anchor_state(boot_table[boot]['end']) 841 else: 842 # Assertion: We trust that the state data was made fresh by the 843 # caller. 844 845 gp.qprintn() 846 847 if boot_table[boot]['method_type'] == "keyword": 848 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 849 boot_table[boot]['method'], 850 quiet=quiet) 851 852 if boot_table[boot]['bmc_reboot']: 853 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 854 plug_in_setup() 855 rc, shell_rc, failed_plug_in_name = \ 856 grpi.rprocess_plug_in_packages(call_point="post_reboot") 857 if rc != 0: 858 error_message = "Plug-in failed with non-zero return code.\n" 859 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 860 set_default_siguser1() 861 BuiltIn().fail(gp.sprint_error(error_message)) 862 else: 863 match_state = st.anchor_state(state) 864 del match_state['epoch_seconds'] 865 # Wait for the state to change in any way. 866 st.wait_state(match_state, wait_time=state_change_timeout, 867 interval="10 seconds", invert=1) 868 869 gp.qprintn() 870 if boot_table[boot]['end']['chassis'] == "Off": 871 boot_timeout = power_off_timeout 872 else: 873 boot_timeout = power_on_timeout 874 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 875 interval="10 seconds") 876 877 plug_in_setup() 878 rc, shell_rc, failed_plug_in_name = \ 879 grpi.rprocess_plug_in_packages(call_point="post_boot") 880 if rc != 0: 881 error_message = "Plug-in failed with non-zero return code.\n" +\ 882 gp.sprint_var(rc, fmt=gp.hexa()) 883 set_default_siguser1() 884 BuiltIn().fail(gp.sprint_error(error_message)) 885 886 # Restore original sigusr1 handler. 887 set_default_siguser1() 888 889 890def test_loop_body(): 891 r""" 892 The main loop body for the loop in main_py. 893 894 Description of arguments: 895 boot_count The iteration number (starts at 1). 896 """ 897 898 global boot_count 899 global state 900 global next_boot 901 global boot_success 902 global boot_end_time 903 904 gp.qprintn() 905 906 next_boot = select_boot() 907 if next_boot == "": 908 return True 909 910 boot_count += 1 911 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 912 913 pre_boot_plug_in_setup() 914 915 cmd_buf = ["run_boot", next_boot] 916 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 917 if boot_status == "FAIL": 918 gp.qprint(msg) 919 920 gp.qprintn() 921 if boot_status == "PASS": 922 boot_success = 1 923 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 924 + "\" succeeded.") 925 else: 926 boot_success = 0 927 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 928 + "\" failed.") 929 930 # Set boot_end_time for use by plug-ins. 931 boot_end_time = completion_msg[1:33] 932 gp.qprint_var(boot_end_time) 933 934 gp.qprint(completion_msg) 935 936 boot_results.update(next_boot, boot_status) 937 938 plug_in_setup() 939 # NOTE: A post_test_case call point failure is NOT counted as a boot 940 # failure. 941 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 942 call_point='post_test_case', stop_on_plug_in_failure=0) 943 944 plug_in_setup() 945 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 946 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 947 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 948 if ffdc_check == "All" or\ 949 shell_rc == dump_ffdc_rc(): 950 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 951 if status != 'PASS': 952 gp.qprint_error("Call to my_ffdc failed.\n") 953 # Leave a record for caller that "soft" errors occurred. 954 soft_errors = 1 955 gpu.save_plug_in_value(soft_errors, pgm_name) 956 957 if delete_errlogs: 958 # We need to purge error logs between boots or they build up. 959 grk.run_key(delete_errlogs_cmd, ignore=1) 960 961 boot_results.print_report() 962 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 963 964 plug_in_setup() 965 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 966 call_point='stop_check', shell_rc=stop_test_rc(), 967 stop_on_non_zero_rc=1) 968 if shell_rc == stop_test_rc(): 969 message = "Stopping as requested by user.\n" 970 gp.qprint_time(message) 971 BuiltIn().fail(message) 972 973 # This should help prevent ConnectionErrors. 974 grk.run_key_u("Close All Connections") 975 976 return True 977 978 979def obmc_boot_test_teardown(): 980 r""" 981 Clean up after the main keyword. 982 """ 983 gp.qprint_executing() 984 985 if ga.psutil_imported: 986 ga.terminate_descendants() 987 988 if cp_setup_called: 989 plug_in_setup() 990 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 991 call_point='cleanup', stop_on_plug_in_failure=0) 992 993 if 'boot_results_file_path' in globals(): 994 # Save boot_results and boot_history objects to a file in case they are 995 # needed again. 996 gp.qprint_timen("Saving boot_results to the following path.") 997 gp.qprint_var(boot_results_file_path) 998 pickle.dump((boot_results, boot_history), 999 open(boot_results_file_path, 'wb'), 1000 pickle.HIGHEST_PROTOCOL) 1001 1002 global save_stack 1003 # Restore any global values saved on the save_stack. 1004 for parm_name in main_func_parm_list: 1005 # Get the parm_value if it was saved on the stack. 1006 try: 1007 parm_value = save_stack.pop(parm_name) 1008 except BaseException: 1009 # If it was not saved, no further action is required. 1010 continue 1011 1012 # Restore the saved value. 1013 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1014 "}\", parm_value)" 1015 gp.dpissuing(cmd_buf) 1016 exec(cmd_buf) 1017 1018 gp.dprintn(save_stack.sprint_obj()) 1019 1020 1021def test_teardown(): 1022 r""" 1023 Clean up after this test case. 1024 """ 1025 1026 gp.qprintn() 1027 gp.qprint_executing() 1028 1029 if ga.psutil_imported: 1030 ga.terminate_descendants() 1031 1032 cmd_buf = ["Print Error", 1033 "A keyword timeout occurred ending this program.\n"] 1034 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1035 1036 gp.qprint_pgm_footer() 1037 1038 1039def post_stack(): 1040 r""" 1041 Process post_stack plug-in programs. 1042 """ 1043 1044 if not call_post_stack_plug: 1045 # The caller does not wish to have post_stack plug-in processing done. 1046 return 1047 1048 global boot_success 1049 1050 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1051 pre_boot_plug_in_setup() 1052 # For the purposes of the following plug-ins, mark the "boot" as a success. 1053 boot_success = 1 1054 plug_in_setup() 1055 rc, shell_rc, failed_plug_in_name, history =\ 1056 grpi.rprocess_plug_in_packages(call_point='post_stack', 1057 stop_on_plug_in_failure=0, 1058 return_history=True) 1059 for doing_msg in history: 1060 update_boot_history(boot_history, doing_msg, max_boot_history) 1061 if rc != 0: 1062 boot_success = 0 1063 1064 plug_in_setup() 1065 rc, shell_rc, failed_plug_in_name =\ 1066 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1067 shell_rc=dump_ffdc_rc(), 1068 stop_on_plug_in_failure=1, 1069 stop_on_non_zero_rc=1) 1070 if shell_rc == dump_ffdc_rc(): 1071 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1072 if status != 'PASS': 1073 gp.qprint_error("Call to my_ffdc failed.\n") 1074 # Leave a record for caller that "soft" errors occurred. 1075 soft_errors = 1 1076 gpu.save_plug_in_value(soft_errors, pgm_name) 1077 1078 plug_in_setup() 1079 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1080 call_point='stop_check', shell_rc=stop_test_rc(), 1081 stop_on_non_zero_rc=1) 1082 if shell_rc == stop_test_rc(): 1083 message = "Stopping as requested by user.\n" 1084 gp.qprint_time(message) 1085 BuiltIn().fail(message) 1086 1087 1088def obmc_boot_test_py(loc_boot_stack=None, 1089 loc_stack_mode=None, 1090 loc_quiet=None): 1091 r""" 1092 Do main program processing. 1093 """ 1094 1095 global save_stack 1096 1097 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1098 1099 gp.dprintn() 1100 # Process function parms. 1101 for parm_name in main_func_parm_list: 1102 # Get parm's value. 1103 parm_value = eval("loc_" + parm_name) 1104 gp.dpvars(parm_name, parm_value) 1105 1106 if parm_value is not None: 1107 # Save the global value on a stack. 1108 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1109 parm_name + "}\"), \"" + parm_name + "\")" 1110 gp.dpissuing(cmd_buf) 1111 exec(cmd_buf) 1112 1113 # Set the global value to the passed value. 1114 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1115 "}\", loc_" + parm_name + ")" 1116 gp.dpissuing(cmd_buf) 1117 exec(cmd_buf) 1118 1119 gp.dprintn(save_stack.sprint_obj()) 1120 1121 setup() 1122 1123 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1124 1125 if ffdc_only: 1126 gp.qprint_timen("Caller requested ffdc_only.") 1127 if do_pre_boot_plug_in_setup: 1128 pre_boot_plug_in_setup() 1129 grk.run_key_u("my_ffdc") 1130 return 1131 1132 if delete_errlogs: 1133 # Delete errlogs prior to doing any boot tests. 1134 grk.run_key(delete_errlogs_cmd, ignore=1) 1135 1136 # Process caller's boot_stack. 1137 while (len(boot_stack) > 0): 1138 test_loop_body() 1139 1140 gp.qprint_timen("Finished processing stack.") 1141 1142 post_stack() 1143 1144 # Process caller's boot_list. 1145 if len(boot_list) > 0: 1146 for ix in range(1, max_num_tests + 1): 1147 test_loop_body() 1148 1149 gp.qprint_timen("Completed all requested boot tests.") 1150 1151 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1152 new_fail = boot_fail - init_boot_fail 1153 if new_fail > boot_fail_threshold: 1154 error_message = "Boot failures exceed the boot failure" +\ 1155 " threshold:\n" +\ 1156 gp.sprint_var(new_fail) +\ 1157 gp.sprint_var(boot_fail_threshold) 1158 BuiltIn().fail(gp.sprint_error(error_message)) 1159